{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.17142857142857143,
  "eval_steps": 500,
  "global_step": 150,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 2700.4271850585938,
      "cov_mean": -6.0587970438064076e-05,
      "cov_std": 0.35307812318205833,
      "entropy": 0.36962890625,
      "epoch": 0.001142857142857143,
      "grad_norm": 0.4682573080062866,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": -0.0446,
      "reward": 0.7604166893288493,
      "reward_std": 0.4268697127699852,
      "rewards/accuracy_reward": 0.25000001303851604,
      "rewards/format_reward": 0.5104166669771075,
      "step": 1,
      "w_high_ratio": 0.21827427297830582,
      "w_low_ratio": 0.03724556043744087,
      "w_max": 2.315404176712036,
      "w_mean": 1.47113436460495,
      "w_min": 0.0,
      "w_std": 0.2791289445012808
    },
    {
      "completion_length": 3127.3958435058594,
      "cov_mean": -2.155053698515985e-05,
      "cov_std": 0.310540571808815,
      "entropy": 0.353515625,
      "epoch": 0.002285714285714286,
      "grad_norm": 0.534198522567749,
      "kl": 0.0,
      "learning_rate": 6.666666666666667e-08,
      "loss": 0.0058,
      "reward": 0.6458333637565374,
      "reward_std": 0.4249730706214905,
      "rewards/accuracy_reward": 0.2812500102445483,
      "rewards/format_reward": 0.3645833386108279,
      "step": 2,
      "w_high_ratio": 0.05722124548628926,
      "w_low_ratio": 0.036368744214996696,
      "w_max": 1.8768170773983002,
      "w_mean": 1.2113382518291473,
      "w_min": 0.0,
      "w_std": 0.19011373445391655
    },
    {
      "completion_length": 3699.729248046875,
      "cov_mean": -7.181393357313937e-05,
      "cov_std": 0.2876722402870655,
      "entropy": 0.458984375,
      "epoch": 0.0034285714285714284,
      "grad_norm": 0.3820074200630188,
      "kl": 4.845857620239258e-05,
      "learning_rate": 1.3333333333333334e-07,
      "loss": 0.0581,
      "reward": 0.250000006519258,
      "reward_std": 0.392750509083271,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.17708334233611822,
      "step": 3,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03734566690400243,
      "w_max": 1.4001508057117462,
      "w_mean": 1.0752681195735931,
      "w_min": 0.0,
      "w_std": 0.16775447502732277
    },
    {
      "completion_length": 2261.197998046875,
      "cov_mean": 2.3754174435453024e-05,
      "cov_std": 0.37356993183493614,
      "entropy": 0.3896484375,
      "epoch": 0.004571428571428572,
      "grad_norm": 0.6735726594924927,
      "kl": 3.325939178466797e-05,
      "learning_rate": 2e-07,
      "loss": -0.0291,
      "reward": 0.9166666865348816,
      "reward_std": 0.4729222096502781,
      "rewards/accuracy_reward": 0.19791667070239782,
      "rewards/format_reward": 0.7187500223517418,
      "step": 4,
      "w_high_ratio": 0.1157110151834786,
      "w_low_ratio": 0.03791455435566604,
      "w_max": 2.3952889442443848,
      "w_mean": 1.526582419872284,
      "w_min": 0.0,
      "w_std": 0.27415894344449043
    },
    {
      "completion_length": 3429.5313720703125,
      "cov_mean": 6.268694096434047e-05,
      "cov_std": 0.43890176713466644,
      "entropy": 0.45556640625,
      "epoch": 0.005714285714285714,
      "grad_norm": 0.4359005391597748,
      "kl": 4.1544437408447266e-05,
      "learning_rate": 2.6666666666666667e-07,
      "loss": -0.0158,
      "reward": 0.385416679084301,
      "reward_std": 0.4654032774269581,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.3437500074505806,
      "step": 5,
      "w_high_ratio": 0.02077934332191944,
      "w_low_ratio": 0.05408582789823413,
      "w_max": 1.6484719216823578,
      "w_mean": 1.1780387163162231,
      "w_min": 1.1194353007380611e-33,
      "w_std": 0.2486402541399002
    },
    {
      "completion_length": 3461.1875610351562,
      "cov_mean": -5.967591278022155e-06,
      "cov_std": 0.49887532368302345,
      "entropy": 0.46533203125,
      "epoch": 0.006857142857142857,
      "grad_norm": 0.3651033937931061,
      "kl": 4.723668098449707e-05,
      "learning_rate": 3.333333333333333e-07,
      "loss": 0.0963,
      "reward": 0.3750000027939677,
      "reward_std": 0.4880600869655609,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.281250006519258,
      "step": 6,
      "w_high_ratio": 0.09943684190511703,
      "w_low_ratio": 0.05423067696392536,
      "w_max": 2.1359716653823853,
      "w_mean": 1.2522149085998535,
      "w_min": 0.0,
      "w_std": 0.3030992951244116
    },
    {
      "completion_length": 3263.5834350585938,
      "cov_mean": 1.3439643225865439e-05,
      "cov_std": 0.48358847945928574,
      "entropy": 0.3818359375,
      "epoch": 0.008,
      "grad_norm": 0.30507004261016846,
      "kl": 1.7881393432617188e-05,
      "learning_rate": 4e-07,
      "loss": -0.0382,
      "reward": 0.895833358168602,
      "reward_std": 0.5626667812466621,
      "rewards/accuracy_reward": 0.2604166716337204,
      "rewards/format_reward": 0.6354166865348816,
      "step": 7,
      "w_high_ratio": 0.0625,
      "w_low_ratio": 0.061702484264969826,
      "w_max": 1.6040385067462921,
      "w_mean": 1.2159111201763153,
      "w_min": 0.0,
      "w_std": 0.2597455531358719
    },
    {
      "completion_length": 2891.2396240234375,
      "cov_mean": -2.5899114461935824e-06,
      "cov_std": 0.31939053907990456,
      "entropy": 0.352783203125,
      "epoch": 0.009142857142857144,
      "grad_norm": 0.2681833803653717,
      "kl": 2.5928020477294922e-05,
      "learning_rate": 4.6666666666666666e-07,
      "loss": -0.0366,
      "reward": 0.8645833544433117,
      "reward_std": 0.5030707456171513,
      "rewards/accuracy_reward": 0.38541667722165585,
      "rewards/format_reward": 0.47916667722165585,
      "step": 8,
      "w_high_ratio": 0.008502780459821224,
      "w_low_ratio": 0.032900307793170214,
      "w_max": 1.777650386095047,
      "w_mean": 1.312690258026123,
      "w_min": 0.0,
      "w_std": 0.19663411937654018
    },
    {
      "completion_length": 3367.1146850585938,
      "cov_mean": -1.6302776657539653e-05,
      "cov_std": 0.46739284694194794,
      "entropy": 0.455078125,
      "epoch": 0.010285714285714285,
      "grad_norm": 0.3553212881088257,
      "kl": 4.4405460357666016e-05,
      "learning_rate": 5.333333333333333e-07,
      "loss": -0.0055,
      "reward": 0.4687500074505806,
      "reward_std": 0.5153512582182884,
      "rewards/accuracy_reward": 0.10416667070239782,
      "rewards/format_reward": 0.3645833432674408,
      "step": 9,
      "w_high_ratio": 0.07327797263860703,
      "w_low_ratio": 0.05655699595808983,
      "w_max": 1.8923848271369934,
      "w_mean": 1.2186144888401031,
      "w_min": 0.0,
      "w_std": 0.2829560115933418
    },
    {
      "completion_length": 2910.5209350585938,
      "cov_mean": -4.502756110014161e-05,
      "cov_std": 0.2934259846806526,
      "entropy": 0.34228515625,
      "epoch": 0.011428571428571429,
      "grad_norm": 0.42780593037605286,
      "kl": 2.6166439056396484e-05,
      "learning_rate": 6e-07,
      "loss": -0.1052,
      "reward": 0.645833358168602,
      "reward_std": 0.4341064542531967,
      "rewards/accuracy_reward": 0.18750001024454832,
      "rewards/format_reward": 0.4583333469927311,
      "step": 10,
      "w_high_ratio": 0.10562621057033539,
      "w_low_ratio": 0.03302360652014613,
      "w_max": 2.053061753511429,
      "w_mean": 1.2806267738342285,
      "w_min": 0.0,
      "w_std": 0.18188510835170746
    },
    {
      "completion_length": 3628.7188110351562,
      "cov_mean": 7.874305651967006e-06,
      "cov_std": 0.26691694743931293,
      "entropy": 0.37451171875,
      "epoch": 0.012571428571428572,
      "grad_norm": 0.3640754818916321,
      "kl": 3.072619438171387e-05,
      "learning_rate": 6.666666666666666e-07,
      "loss": 0.028,
      "reward": 0.22916667442768812,
      "reward_std": 0.3545106574892998,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.16666666697710752,
      "step": 11,
      "w_high_ratio": 0.02717638947069645,
      "w_low_ratio": 0.032686853082850575,
      "w_max": 1.4469610452651978,
      "w_mean": 1.1045592427253723,
      "w_min": 0.25,
      "w_std": 0.16260053776204586
    },
    {
      "completion_length": 2600.3334350585938,
      "cov_mean": -2.9306334909051657e-05,
      "cov_std": 0.36532483994960785,
      "entropy": 0.3974609375,
      "epoch": 0.013714285714285714,
      "grad_norm": 0.5842437148094177,
      "kl": 3.3795833587646484e-05,
      "learning_rate": 7.333333333333332e-07,
      "loss": -0.0312,
      "reward": 0.8541667014360428,
      "reward_std": 0.42210913449525833,
      "rewards/accuracy_reward": 0.16666667070239782,
      "rewards/format_reward": 0.6875000298023224,
      "step": 12,
      "w_high_ratio": 0.25580327026546,
      "w_low_ratio": 0.03646009974181652,
      "w_max": 2.4061461091041565,
      "w_mean": 1.5557032227516174,
      "w_min": 0.0,
      "w_std": 0.2952301353216171
    },
    {
      "completion_length": 3168.260498046875,
      "cov_mean": -2.8395811568771023e-07,
      "cov_std": 0.16673796251416206,
      "entropy": 0.3974609375,
      "epoch": 0.014857142857142857,
      "grad_norm": 0.32018211483955383,
      "kl": 3.4928321838378906e-05,
      "learning_rate": 8e-07,
      "loss": 0.013,
      "reward": 0.6666666865348816,
      "reward_std": 0.21763009577989578,
      "rewards/accuracy_reward": 0.21875000558793545,
      "rewards/format_reward": 0.447916679084301,
      "step": 13,
      "w_high_ratio": 0.16240009665489197,
      "w_low_ratio": 0.019987554755061865,
      "w_max": 1.9363721311092377,
      "w_mean": 1.3362610340118408,
      "w_min": 0.25,
      "w_std": 0.13108899258077145
    },
    {
      "completion_length": 3131.4896240234375,
      "cov_mean": 2.226169999630656e-05,
      "cov_std": 0.4046770706772804,
      "entropy": 0.37109375,
      "epoch": 0.016,
      "grad_norm": 0.37404048442840576,
      "kl": 2.765655517578125e-05,
      "learning_rate": 8.666666666666667e-07,
      "loss": -0.0183,
      "reward": 0.697916679084301,
      "reward_std": 0.5398248583078384,
      "rewards/accuracy_reward": 0.23958334140479565,
      "rewards/format_reward": 0.4583333507180214,
      "step": 14,
      "w_high_ratio": 0.04599327966570854,
      "w_low_ratio": 0.04479631967842579,
      "w_max": 1.9890422523021698,
      "w_mean": 1.2714892327785492,
      "w_min": 0.0,
      "w_std": 0.2778457775712013
    },
    {
      "completion_length": 2876.5000610351562,
      "cov_mean": 5.642831865770859e-05,
      "cov_std": 0.2463381662964821,
      "entropy": 0.3603515625,
      "epoch": 0.017142857142857144,
      "grad_norm": 0.3123975694179535,
      "kl": 2.4527311325073242e-05,
      "learning_rate": 9.333333333333333e-07,
      "loss": 0.0189,
      "reward": 0.6145833358168602,
      "reward_std": 0.282865684479475,
      "rewards/accuracy_reward": 0.21875000279396772,
      "rewards/format_reward": 0.3958333395421505,
      "step": 15,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.025731001514941454,
      "w_max": 1.6097791492938995,
      "w_mean": 1.1706343591213226,
      "w_min": 0.25,
      "w_std": 0.15255355089902878
    },
    {
      "completion_length": 3825.5313110351562,
      "cov_mean": 4.868104133493034e-05,
      "cov_std": 0.24393152818083763,
      "entropy": 0.462890625,
      "epoch": 0.018285714285714287,
      "grad_norm": 0.33849093317985535,
      "kl": 3.88026237487793e-05,
      "learning_rate": 1e-06,
      "loss": 0.0219,
      "reward": 0.1458333358168602,
      "reward_std": 0.3029968775808811,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.09375,
      "step": 16,
      "w_high_ratio": 0.016145935282111168,
      "w_low_ratio": 0.030266874469816685,
      "w_max": 1.4482556581497192,
      "w_mean": 1.039628803730011,
      "w_min": 0.25,
      "w_std": 0.12409967929124832
    },
    {
      "completion_length": 2457.187530517578,
      "cov_mean": -5.5577158491360024e-05,
      "cov_std": 0.40591511130332947,
      "entropy": 0.44580078125,
      "epoch": 0.019428571428571427,
      "grad_norm": 0.4333915710449219,
      "kl": 3.7103891372680664e-05,
      "learning_rate": 9.998781585307575e-07,
      "loss": -0.0184,
      "reward": 0.8645833432674408,
      "reward_std": 0.4666195958852768,
      "rewards/accuracy_reward": 0.27083334140479565,
      "rewards/format_reward": 0.5937500149011612,
      "step": 17,
      "w_high_ratio": 0.2576238848268986,
      "w_low_ratio": 0.045077938586473465,
      "w_max": 2.4780974686145782,
      "w_mean": 1.4898322224617004,
      "w_min": 0.25,
      "w_std": 0.27607931941747665
    },
    {
      "completion_length": 3082.572998046875,
      "cov_mean": 2.8908147328365885e-05,
      "cov_std": 0.37330811098217964,
      "entropy": 0.35791015625,
      "epoch": 0.02057142857142857,
      "grad_norm": 0.4184105098247528,
      "kl": 1.868605613708496e-05,
      "learning_rate": 9.99512700102336e-07,
      "loss": -0.0108,
      "reward": 0.5833333432674408,
      "reward_std": 0.35651107877492905,
      "rewards/accuracy_reward": 0.12500000558793545,
      "rewards/format_reward": 0.4583333507180214,
      "step": 18,
      "w_high_ratio": 0.05808360502123833,
      "w_low_ratio": 0.036713168025016785,
      "w_max": 1.845141887664795,
      "w_mean": 1.2742244899272919,
      "w_min": 0.0,
      "w_std": 0.20810226537287235
    },
    {
      "completion_length": 3121.2084350585938,
      "cov_mean": -7.294934039236978e-05,
      "cov_std": 0.43627090007066727,
      "entropy": 0.390625,
      "epoch": 0.021714285714285714,
      "grad_norm": 0.37888550758361816,
      "kl": 3.045797348022461e-05,
      "learning_rate": 9.989038226169207e-07,
      "loss": -0.0515,
      "reward": 0.8437500223517418,
      "reward_std": 0.6305291801691055,
      "rewards/accuracy_reward": 0.37500001676380634,
      "rewards/format_reward": 0.4687500260770321,
      "step": 19,
      "w_high_ratio": 0.07324637286365032,
      "w_low_ratio": 0.05016931891441345,
      "w_max": 1.8255594372749329,
      "w_mean": 1.2571382224559784,
      "w_min": 0.0,
      "w_std": 0.2677953541278839
    },
    {
      "completion_length": 2640.885498046875,
      "cov_mean": 4.735650145448744e-05,
      "cov_std": 0.3673415333032608,
      "entropy": 0.346923828125,
      "epoch": 0.022857142857142857,
      "grad_norm": 0.5563057065010071,
      "kl": 2.5674700736999512e-05,
      "learning_rate": 9.98051855792412e-07,
      "loss": -0.0634,
      "reward": 0.927083358168602,
      "reward_std": 0.4865139201283455,
      "rewards/accuracy_reward": 0.291666679084301,
      "rewards/format_reward": 0.6354166865348816,
      "step": 20,
      "w_high_ratio": 0.1635199710726738,
      "w_low_ratio": 0.04272336233407259,
      "w_max": 2.4736950993537903,
      "w_mean": 1.437729924917221,
      "w_min": 0.0,
      "w_std": 0.2519150599837303
    },
    {
      "completion_length": 2853.541748046875,
      "cov_mean": 2.0583035620802548e-05,
      "cov_std": 0.330572672188282,
      "entropy": 0.4111328125,
      "epoch": 0.024,
      "grad_norm": 1.0757092237472534,
      "kl": 6.0230493545532227e-05,
      "learning_rate": 9.969572609838744e-07,
      "loss": -0.0628,
      "reward": 0.6458333684131503,
      "reward_std": 0.4320429190993309,
      "rewards/accuracy_reward": 0.1875000074505806,
      "rewards/format_reward": 0.4583333535119891,
      "step": 21,
      "w_high_ratio": 0.2964049205183983,
      "w_low_ratio": 0.028110376093536615,
      "w_max": 2.3169990181922913,
      "w_mean": 1.5084502398967743,
      "w_min": 0.0,
      "w_std": 0.2527524419128895
    },
    {
      "completion_length": 1911.4584045410156,
      "cov_mean": 5.60426842639572e-05,
      "cov_std": 0.40649277716875076,
      "entropy": 0.4189453125,
      "epoch": 0.025142857142857144,
      "grad_norm": 0.6271277666091919,
      "kl": 7.963180541992188e-05,
      "learning_rate": 9.956206309337066e-07,
      "loss": -0.0446,
      "reward": 1.0937500447034836,
      "reward_std": 0.43423888459801674,
      "rewards/accuracy_reward": 0.26041666977107525,
      "rewards/format_reward": 0.833333358168602,
      "step": 22,
      "w_high_ratio": 0.31319986283779144,
      "w_low_ratio": 0.043129971250891685,
      "w_max": 2.544324040412903,
      "w_mean": 1.6584191024303436,
      "w_min": 1.0509738482436128e-45,
      "w_std": 0.2851412668824196
    },
    {
      "completion_length": 2764.9896240234375,
      "cov_mean": -5.482907317855279e-05,
      "cov_std": 0.4773280769586563,
      "entropy": 0.364013671875,
      "epoch": 0.026285714285714287,
      "grad_norm": 0.4739457368850708,
      "kl": 5.441904067993164e-05,
      "learning_rate": 9.940426894506606e-07,
      "loss": 0.0215,
      "reward": 0.677083358168602,
      "reward_std": 0.5219395384192467,
      "rewards/accuracy_reward": 0.17708333861082792,
      "rewards/format_reward": 0.5000000149011612,
      "step": 23,
      "w_high_ratio": 0.2707533538341522,
      "w_low_ratio": 0.053491173312067986,
      "w_max": 2.4205015003681183,
      "w_mean": 1.5040302574634552,
      "w_min": 0.0,
      "w_std": 0.36683739349246025
    },
    {
      "completion_length": 2940.791748046875,
      "cov_mean": -2.262868292746134e-07,
      "cov_std": 0.4811120182275772,
      "entropy": 0.36572265625,
      "epoch": 0.027428571428571427,
      "grad_norm": 0.7653826475143433,
      "kl": 7.31348991394043e-05,
      "learning_rate": 9.922242910178859e-07,
      "loss": -0.0493,
      "reward": 0.8750000149011612,
      "reward_std": 0.580329179763794,
      "rewards/accuracy_reward": 0.3125000074505806,
      "rewards/format_reward": 0.5625,
      "step": 24,
      "w_high_ratio": 0.13745100796222687,
      "w_low_ratio": 0.043023983016610146,
      "w_max": 2.282612681388855,
      "w_mean": 1.408221811056137,
      "w_min": 0.0,
      "w_std": 0.30761053785681725
    },
    {
      "completion_length": 2838.385498046875,
      "cov_mean": 5.768927030658233e-05,
      "cov_std": 0.3588615171611309,
      "entropy": 0.43115234375,
      "epoch": 0.02857142857142857,
      "grad_norm": 0.4592779278755188,
      "kl": 0.0001583397388458252,
      "learning_rate": 9.901664203302124e-07,
      "loss": 0.0336,
      "reward": 0.6458333535119891,
      "reward_std": 0.42768918722867966,
      "rewards/accuracy_reward": 0.2083333358168602,
      "rewards/format_reward": 0.4375000102445483,
      "step": 25,
      "w_high_ratio": 0.25193173810839653,
      "w_low_ratio": 0.03785138111561537,
      "w_max": 2.232436418533325,
      "w_mean": 1.3874212205410004,
      "w_min": 0.0,
      "w_std": 0.26341583393514156
    },
    {
      "completion_length": 3263.9271850585938,
      "cov_mean": 5.7232594372180756e-05,
      "cov_std": 0.3583865277469158,
      "entropy": 0.423828125,
      "epoch": 0.029714285714285714,
      "grad_norm": 0.29540950059890747,
      "kl": 2.8073787689208984e-05,
      "learning_rate": 9.878701917609207e-07,
      "loss": -0.0308,
      "reward": 0.6562500298023224,
      "reward_std": 0.30247221142053604,
      "rewards/accuracy_reward": 0.20833334140479565,
      "rewards/format_reward": 0.4479166716337204,
      "step": 26,
      "w_high_ratio": 0.043125174939632416,
      "w_low_ratio": 0.034932715352624655,
      "w_max": 1.7730375826358795,
      "w_mean": 1.1945610046386719,
      "w_min": 0.0,
      "w_std": 0.21625201031565666
    },
    {
      "completion_length": 3234.354248046875,
      "cov_mean": -1.585684123028841e-05,
      "cov_std": 0.3628169037401676,
      "entropy": 0.4287109375,
      "epoch": 0.030857142857142857,
      "grad_norm": 0.23347000777721405,
      "kl": 7.574260234832764e-05,
      "learning_rate": 9.853368487582886e-07,
      "loss": 0.0016,
      "reward": 0.5729166939854622,
      "reward_std": 0.4602612778544426,
      "rewards/accuracy_reward": 0.12500000279396772,
      "rewards/format_reward": 0.447916679084301,
      "step": 27,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.049336991272866726,
      "w_max": 1.539461612701416,
      "w_mean": 1.1640309989452362,
      "w_min": 0.0,
      "w_std": 0.19506899639964104
    },
    {
      "completion_length": 3126.1458740234375,
      "cov_mean": -2.0113498976570554e-05,
      "cov_std": 0.4664968028664589,
      "entropy": 0.40576171875,
      "epoch": 0.032,
      "grad_norm": 0.3652705252170563,
      "kl": 9.274482727050781e-05,
      "learning_rate": 9.825677631722435e-07,
      "loss": 0.0034,
      "reward": 0.7916666865348816,
      "reward_std": 0.5775652155280113,
      "rewards/accuracy_reward": 0.322916679084301,
      "rewards/format_reward": 0.4687500149011612,
      "step": 28,
      "w_high_ratio": 0.0399150624871254,
      "w_low_ratio": 0.05781116522848606,
      "w_max": 1.8517873883247375,
      "w_mean": 1.2485012710094452,
      "w_min": 0.0,
      "w_std": 0.2899218685925007
    },
    {
      "completion_length": 3620.3854370117188,
      "cov_mean": 3.1027989280119073e-06,
      "cov_std": 0.3902217298746109,
      "entropy": 0.42919921875,
      "epoch": 0.03314285714285714,
      "grad_norm": 0.42134925723075867,
      "kl": 0.00019466876983642578,
      "learning_rate": 9.795644345114794e-07,
      "loss": 0.054,
      "reward": 0.26041667722165585,
      "reward_std": 0.39330877363681793,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.21875000558793545,
      "step": 29,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.05211624875664711,
      "w_max": 1.389526218175888,
      "w_mean": 1.1012998819351196,
      "w_min": 0.25,
      "w_std": 0.21629613637924194
    },
    {
      "completion_length": 3127.28125,
      "cov_mean": 7.65450022299774e-06,
      "cov_std": 0.39614470303058624,
      "entropy": 0.37109375,
      "epoch": 0.03428571428571429,
      "grad_norm": 0.3341064751148224,
      "kl": 0.0004119873046875,
      "learning_rate": 9.76328489131448e-07,
      "loss": 0.0409,
      "reward": 0.7916666865348816,
      "reward_std": 0.536014050245285,
      "rewards/accuracy_reward": 0.2708333395421505,
      "rewards/format_reward": 0.5208333432674408,
      "step": 30,
      "w_high_ratio": 0.09042909741401672,
      "w_low_ratio": 0.04219994880259037,
      "w_max": 2.0287185609340668,
      "w_mean": 1.2833797633647919,
      "w_min": 0.25,
      "w_std": 0.2540416121482849
    },
    {
      "completion_length": 3311.0418090820312,
      "cov_mean": 1.0463507578606368e-05,
      "cov_std": 0.2679142467677593,
      "entropy": 0.392578125,
      "epoch": 0.03542857142857143,
      "grad_norm": 0.34394925832748413,
      "kl": 0.00025856494903564453,
      "learning_rate": 9.728616793536587e-07,
      "loss": 0.008,
      "reward": 0.5104166772216558,
      "reward_std": 0.41994429379701614,
      "rewards/accuracy_reward": 0.19791666977107525,
      "rewards/format_reward": 0.3125000027939677,
      "step": 31,
      "w_high_ratio": 0.013557232916355133,
      "w_low_ratio": 0.029067183146253228,
      "w_max": 1.5240460634231567,
      "w_mean": 1.1379797160625458,
      "w_min": 0.0,
      "w_std": 0.1539093293249607
    },
    {
      "completion_length": 3302.9793090820312,
      "cov_mean": -3.0240359592426103e-05,
      "cov_std": 0.38900837302207947,
      "entropy": 0.4296875,
      "epoch": 0.036571428571428574,
      "grad_norm": 0.3267619013786316,
      "kl": 0.0003151893615722656,
      "learning_rate": 9.69165882516764e-07,
      "loss": 0.0316,
      "reward": 0.708333333954215,
      "reward_std": 0.4763314798474312,
      "rewards/accuracy_reward": 0.29166666977107525,
      "rewards/format_reward": 0.4166666744276881,
      "step": 32,
      "w_high_ratio": 0.03662256337702274,
      "w_low_ratio": 0.04006141540594399,
      "w_max": 1.6614282727241516,
      "w_mean": 1.1914446651935577,
      "w_min": 0.0,
      "w_std": 0.22214871272444725
    },
    {
      "completion_length": 3574.9063720703125,
      "cov_mean": -1.1321862984914333e-05,
      "cov_std": 0.35655253008008003,
      "entropy": 0.375,
      "epoch": 0.037714285714285714,
      "grad_norm": 0.22990448772907257,
      "kl": 0.00036966800689697266,
      "learning_rate": 9.65243099959949e-07,
      "loss": -0.0046,
      "reward": 0.614583358168602,
      "reward_std": 0.5987343490123749,
      "rewards/accuracy_reward": 0.2395833432674408,
      "rewards/format_reward": 0.375,
      "step": 33,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.050560859963297844,
      "w_max": 1.4202565550804138,
      "w_mean": 1.1010453402996063,
      "w_min": 9.80908925027372e-45,
      "w_std": 0.22285480797290802
    },
    {
      "completion_length": 2701.3021545410156,
      "cov_mean": -9.209951758748502e-05,
      "cov_std": 0.40857063978910446,
      "entropy": 0.455078125,
      "epoch": 0.038857142857142854,
      "grad_norm": 0.49724769592285156,
      "kl": 0.0010764598846435547,
      "learning_rate": 9.610954559391704e-07,
      "loss": -0.0999,
      "reward": 0.8229166883975267,
      "reward_std": 0.44509488344192505,
      "rewards/accuracy_reward": 0.32291668001562357,
      "rewards/format_reward": 0.5000000158324838,
      "step": 34,
      "w_high_ratio": 0.255466103553772,
      "w_low_ratio": 0.058159707114100456,
      "w_max": 1.966733694076538,
      "w_mean": 1.445367842912674,
      "w_min": 0.0,
      "w_std": 0.2951628230512142
    },
    {
      "completion_length": 3312.5209350585938,
      "cov_mean": -2.75732190857525e-06,
      "cov_std": 0.4042964428663254,
      "entropy": 0.43115234375,
      "epoch": 0.04,
      "grad_norm": 0.5397042036056519,
      "kl": 0.0008138418197631836,
      "learning_rate": 9.567251964768342e-07,
      "loss": -0.0246,
      "reward": 0.6041666716337204,
      "reward_std": 0.5854796469211578,
      "rewards/accuracy_reward": 0.2604166753590107,
      "rewards/format_reward": 0.3437500111758709,
      "step": 35,
      "w_high_ratio": 0.05638222396373749,
      "w_low_ratio": 0.05512247420847416,
      "w_max": 2.192526876926422,
      "w_mean": 1.2277101576328278,
      "w_min": 5.989329002727702e-37,
      "w_std": 0.2675026059150696
    },
    {
      "completion_length": 3587.6354370117188,
      "cov_mean": -1.1433875897637336e-05,
      "cov_std": 0.2749627083539963,
      "entropy": 0.48095703125,
      "epoch": 0.04114285714285714,
      "grad_norm": 0.2379421591758728,
      "kl": 0.000979304313659668,
      "learning_rate": 9.521346881455354e-07,
      "loss": -0.0089,
      "reward": 0.22916666977107525,
      "reward_std": 0.3393310159444809,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.17708334140479565,
      "step": 36,
      "w_high_ratio": 0.0930290725082159,
      "w_low_ratio": 0.03267599269747734,
      "w_max": 1.6210260689258575,
      "w_mean": 1.149814784526825,
      "w_min": 0.25,
      "w_std": 0.14480087533593178
    },
    {
      "completion_length": 3585.6458740234375,
      "cov_mean": -6.411561662389431e-05,
      "cov_std": 0.3158421888947487,
      "entropy": 0.42236328125,
      "epoch": 0.04228571428571429,
      "grad_norm": 0.2652978301048279,
      "kl": 0.0006819963455200195,
      "learning_rate": 9.473264167865171e-07,
      "loss": 0.0058,
      "reward": 0.23958334419876337,
      "reward_std": 0.20556553453207016,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.21875000558793545,
      "step": 37,
      "w_high_ratio": 0.054578784853219986,
      "w_low_ratio": 0.03975548921152949,
      "w_max": 1.6195516288280487,
      "w_mean": 1.1414334774017334,
      "w_min": 0.0,
      "w_std": 0.17483297176659107
    },
    {
      "completion_length": 3557.1563110351562,
      "cov_mean": 1.9627160781965358e-05,
      "cov_std": 0.1310347281396389,
      "entropy": 0.4443359375,
      "epoch": 0.04342857142857143,
      "grad_norm": 0.18166524171829224,
      "kl": 0.00045049190521240234,
      "learning_rate": 9.42302986163543e-07,
      "loss": 0.0055,
      "reward": 0.2916666716337204,
      "reward_std": 0.20568452775478363,
      "rewards/accuracy_reward": 0.1354166716337204,
      "rewards/format_reward": 0.15625,
      "step": 38,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.013907193206250668,
      "w_max": 1.2318150103092194,
      "w_mean": 1.0593293607234955,
      "w_min": 0.5,
      "w_std": 0.0720198005437851
    },
    {
      "completion_length": 3032.9688110351562,
      "cov_mean": 3.0818391678621992e-06,
      "cov_std": 0.21192274242639542,
      "entropy": 0.361083984375,
      "epoch": 0.044571428571428574,
      "grad_norm": 0.19133131206035614,
      "kl": 0.001926124095916748,
      "learning_rate": 9.370671165529144e-07,
      "loss": 0.0176,
      "reward": 0.760416679084301,
      "reward_std": 0.19024790823459625,
      "rewards/accuracy_reward": 0.2604166669771075,
      "rewards/format_reward": 0.5000000074505806,
      "step": 39,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.031908176839351654,
      "w_max": 1.4641892611980438,
      "w_mean": 1.1698184311389923,
      "w_min": 0.5,
      "w_std": 0.11090587638318539
    },
    {
      "completion_length": 3034.4375610351562,
      "cov_mean": -2.096771822834853e-05,
      "cov_std": 0.2704497389495373,
      "entropy": 0.3974609375,
      "epoch": 0.045714285714285714,
      "grad_norm": 0.2450522482395172,
      "kl": 0.0029773712158203125,
      "learning_rate": 9.316216432703916e-07,
      "loss": 0.0304,
      "reward": 0.6145833469927311,
      "reward_std": 0.3184027150273323,
      "rewards/accuracy_reward": 0.16666666697710752,
      "rewards/format_reward": 0.447916679084301,
      "step": 40,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03790469467639923,
      "w_max": 1.6503434479236603,
      "w_mean": 1.2544237673282623,
      "w_min": 0.0,
      "w_std": 0.16268039494752884
    },
    {
      "completion_length": 3323.0313110351562,
      "cov_mean": 1.6318189409503248e-05,
      "cov_std": 0.3728119507431984,
      "entropy": 0.37890625,
      "epoch": 0.046857142857142854,
      "grad_norm": 0.25028058886528015,
      "kl": 0.0005707740783691406,
      "learning_rate": 9.259695151358214e-07,
      "loss": 0.0052,
      "reward": 0.4166666716337204,
      "reward_std": 0.42720501869916916,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.3541666716337204,
      "step": 41,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.04678645543754101,
      "w_max": 1.4909087419509888,
      "w_mean": 1.1125215888023376,
      "w_min": 0.0,
      "w_std": 0.20876972749829292
    },
    {
      "completion_length": 2951.6458740234375,
      "cov_mean": 5.0857947826443706e-05,
      "cov_std": 0.20116684958338737,
      "entropy": 0.45947265625,
      "epoch": 0.048,
      "grad_norm": 0.20272254943847656,
      "kl": 0.0004711151123046875,
      "learning_rate": 9.20113792876298e-07,
      "loss": 0.0143,
      "reward": 0.40625000558793545,
      "reward_std": 0.2362503558397293,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.3750000027939677,
      "step": 42,
      "w_high_ratio": 0.125,
      "w_low_ratio": 0.02315131900832057,
      "w_max": 1.6171163022518158,
      "w_mean": 1.2283784747123718,
      "w_min": 0.5,
      "w_std": 0.10980619117617607
    },
    {
      "completion_length": 3174.875,
      "cov_mean": 4.1157167999017474e-05,
      "cov_std": 0.3693430423736572,
      "entropy": 0.396484375,
      "epoch": 0.04914285714285714,
      "grad_norm": 0.2846878170967102,
      "kl": 0.0015649795532226562,
      "learning_rate": 9.140576474687263e-07,
      "loss": 0.0363,
      "reward": 0.5000000111758709,
      "reward_std": 0.39679284393787384,
      "rewards/accuracy_reward": 0.17708334140479565,
      "rewards/format_reward": 0.32291666977107525,
      "step": 43,
      "w_high_ratio": 0.0364043265581131,
      "w_low_ratio": 0.052212903276085854,
      "w_max": 1.6643747389316559,
      "w_mean": 1.1576766669750214,
      "w_min": 0.0,
      "w_std": 0.22118561156094074
    },
    {
      "completion_length": 2870.6771697998047,
      "cov_mean": -0.00011349500164214987,
      "cov_std": 0.38516905158758163,
      "entropy": 0.392578125,
      "epoch": 0.05028571428571429,
      "grad_norm": 0.36889317631721497,
      "kl": 0.001346588134765625,
      "learning_rate": 9.078043584226815e-07,
      "loss": -0.016,
      "reward": 0.7708333469927311,
      "reward_std": 0.4629998579621315,
      "rewards/accuracy_reward": 0.25000000558793545,
      "rewards/format_reward": 0.520833333954215,
      "step": 44,
      "w_high_ratio": 0.16294695064425468,
      "w_low_ratio": 0.04376620473340154,
      "w_max": 1.860895425081253,
      "w_mean": 1.262012630701065,
      "w_min": 0.0,
      "w_std": 0.2177225835621357
    },
    {
      "completion_length": 3653.2084350585938,
      "cov_mean": 2.4295502612403652e-05,
      "cov_std": 0.34152911603450775,
      "entropy": 0.41552734375,
      "epoch": 0.05142857142857143,
      "grad_norm": 0.3034785985946655,
      "kl": 0.0036156177520751953,
      "learning_rate": 9.013573120044966e-07,
      "loss": 0.0183,
      "reward": 0.4375,
      "reward_std": 0.47292545437812805,
      "rewards/accuracy_reward": 0.17708333861082792,
      "rewards/format_reward": 0.26041667349636555,
      "step": 45,
      "w_high_ratio": 0.0399763397872448,
      "w_low_ratio": 0.03463862743228674,
      "w_max": 1.478510558605194,
      "w_mean": 1.0995357930660248,
      "w_min": 0.0,
      "w_std": 0.18589595332741737
    },
    {
      "completion_length": 3480.4583740234375,
      "cov_mean": -8.883081477506494e-06,
      "cov_std": 0.35656186379492283,
      "entropy": 0.482421875,
      "epoch": 0.052571428571428575,
      "grad_norm": 0.43441376090049744,
      "kl": 0.0010223388671875,
      "learning_rate": 8.9471999940354e-07,
      "loss": 0.1236,
      "reward": 0.2708333348855376,
      "reward_std": 0.3359568640589714,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.22916667442768812,
      "step": 46,
      "w_high_ratio": 0.09412252902984619,
      "w_low_ratio": 0.039234326453879476,
      "w_max": 1.7104443907737732,
      "w_mean": 1.1970912516117096,
      "w_min": 0.0,
      "w_std": 0.22640072740614414
    },
    {
      "completion_length": 3027.9375610351562,
      "cov_mean": -2.4718745407881215e-05,
      "cov_std": 0.26147888600826263,
      "entropy": 0.38720703125,
      "epoch": 0.053714285714285714,
      "grad_norm": 0.29539671540260315,
      "kl": 0.0011167526245117188,
      "learning_rate": 8.878960148416747e-07,
      "loss": -0.0436,
      "reward": 0.8125000149011612,
      "reward_std": 0.4830247238278389,
      "rewards/accuracy_reward": 0.291666679084301,
      "rewards/format_reward": 0.5208333432674408,
      "step": 47,
      "w_high_ratio": 0.11265619844198227,
      "w_low_ratio": 0.028757336549460888,
      "w_max": 1.9260917007923126,
      "w_mean": 1.3850333988666534,
      "w_min": 0.25,
      "w_std": 0.19246497750282288
    },
    {
      "completion_length": 3071.947998046875,
      "cov_mean": 6.93041113208892e-05,
      "cov_std": 0.44929926097393036,
      "entropy": 0.3916015625,
      "epoch": 0.054857142857142854,
      "grad_norm": 0.429691344499588,
      "kl": 0.0033349990844726562,
      "learning_rate": 8.808890536269229e-07,
      "loss": 0.0121,
      "reward": 0.6250000074505806,
      "reward_std": 0.5002652183175087,
      "rewards/accuracy_reward": 0.21875000651925802,
      "rewards/format_reward": 0.40625001303851604,
      "step": 48,
      "w_high_ratio": 0.11854390799999237,
      "w_low_ratio": 0.049413095228374004,
      "w_max": 2.040738523006439,
      "w_mean": 1.2706988453865051,
      "w_min": 0.0,
      "w_std": 0.28789742290973663
    },
    {
      "completion_length": 2590.0208740234375,
      "cov_mean": 1.1296036518615438e-05,
      "cov_std": 0.44136329740285873,
      "entropy": 0.361328125,
      "epoch": 0.056,
      "grad_norm": 0.3813425898551941,
      "kl": 0.006386756896972656,
      "learning_rate": 8.737029101523929e-07,
      "loss": 0.0099,
      "reward": 0.8541667014360428,
      "reward_std": 0.5753171592950821,
      "rewards/accuracy_reward": 0.3020833358168602,
      "rewards/format_reward": 0.5520833507180214,
      "step": 49,
      "w_high_ratio": 0.08562466688454151,
      "w_low_ratio": 0.04912099055945873,
      "w_max": 2.1155774295330048,
      "w_mean": 1.313397854566574,
      "w_min": 1.0880903909928003e-39,
      "w_std": 0.2910540699958801
    },
    {
      "completion_length": 3252.7188110351562,
      "cov_mean": -5.4081367125036195e-05,
      "cov_std": 0.2936253622174263,
      "entropy": 0.341796875,
      "epoch": 0.05714285714285714,
      "grad_norm": 0.19821792840957642,
      "kl": 0.0009126663208007812,
      "learning_rate": 8.663414758415478e-07,
      "loss": 0.0282,
      "reward": 0.5937500204890966,
      "reward_std": 0.34640391170978546,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.34375000558793545,
      "step": 50,
      "w_high_ratio": 0.0625,
      "w_low_ratio": 0.038133963476866484,
      "w_max": 1.6234075427055359,
      "w_mean": 1.1869377791881561,
      "w_min": 0.25,
      "w_std": 0.1625902745872736
    },
    {
      "completion_length": 2461.135498046875,
      "cov_mean": 2.296896900588763e-05,
      "cov_std": 0.34488509595394135,
      "entropy": 0.43603515625,
      "epoch": 0.05828571428571429,
      "grad_norm": 0.37042999267578125,
      "kl": 0.0047626495361328125,
      "learning_rate": 8.588087370409302e-07,
      "loss": 0.0031,
      "reward": 0.6875000149011612,
      "reward_std": 0.3103678971529007,
      "rewards/accuracy_reward": 0.14583333767950535,
      "rewards/format_reward": 0.5416666716337204,
      "step": 51,
      "w_high_ratio": 0.012511095963418484,
      "w_low_ratio": 0.04256652761250734,
      "w_max": 1.5135074257850647,
      "w_mean": 1.154930055141449,
      "w_min": 0.25,
      "w_std": 0.20846375823020935
    },
    {
      "completion_length": 3097.0000610351562,
      "cov_mean": 1.4503702914225869e-06,
      "cov_std": 0.42206430435180664,
      "entropy": 0.39306640625,
      "epoch": 0.05942857142857143,
      "grad_norm": 0.7796747088432312,
      "kl": 0.0031175613403320312,
      "learning_rate": 8.511087728614862e-07,
      "loss": -0.0444,
      "reward": 0.7083333684131503,
      "reward_std": 0.5597149804234505,
      "rewards/accuracy_reward": 0.2812500074505806,
      "rewards/format_reward": 0.4270833386108279,
      "step": 52,
      "w_high_ratio": 0.1064748540520668,
      "w_low_ratio": 0.0502536753192544,
      "w_max": 1.7458641231060028,
      "w_mean": 1.2538467645645142,
      "w_min": 0.0,
      "w_std": 0.25843533128499985
    },
    {
      "completion_length": 2980.0833740234375,
      "cov_mean": -1.4053926861379296e-05,
      "cov_std": 0.5106082037091255,
      "entropy": 0.4208984375,
      "epoch": 0.060571428571428575,
      "grad_norm": 0.43870407342910767,
      "kl": 0.0011355876922607422,
      "learning_rate": 8.432457529696548e-07,
      "loss": -0.0332,
      "reward": 0.8229167014360428,
      "reward_std": 0.6252040863037109,
      "rewards/accuracy_reward": 0.2812500074505806,
      "rewards/format_reward": 0.5416666865348816,
      "step": 53,
      "w_high_ratio": 0.15575578436255455,
      "w_low_ratio": 0.04851931845769286,
      "w_max": 2.0333048701286316,
      "w_mean": 1.3303064107894897,
      "w_min": 0.0,
      "w_std": 0.3132343143224716
    },
    {
      "completion_length": 3125.135498046875,
      "cov_mean": 2.5879786335281096e-05,
      "cov_std": 0.4214501827955246,
      "entropy": 0.38623046875,
      "epoch": 0.061714285714285715,
      "grad_norm": 0.41025811433792114,
      "kl": 0.0010042190551757812,
      "learning_rate": 8.352239353294194e-07,
      "loss": -0.0491,
      "reward": 0.9479167014360428,
      "reward_std": 0.6265207231044769,
      "rewards/accuracy_reward": 0.416666679084301,
      "rewards/format_reward": 0.5312500149011612,
      "step": 54,
      "w_high_ratio": 0.045562680810689926,
      "w_low_ratio": 0.05255642905831337,
      "w_max": 1.7965390384197235,
      "w_mean": 1.294397234916687,
      "w_min": 0.0,
      "w_std": 0.2626797705888748
    },
    {
      "completion_length": 3290.5521240234375,
      "cov_mean": 6.297564050328219e-05,
      "cov_std": 0.3725521042943001,
      "entropy": 0.3955078125,
      "epoch": 0.06285714285714286,
      "grad_norm": 0.3483085632324219,
      "kl": 0.0009459257125854492,
      "learning_rate": 8.270476638965461e-07,
      "loss": 0.0117,
      "reward": 0.6979166716337204,
      "reward_std": 0.5277387201786041,
      "rewards/accuracy_reward": 0.2604166716337204,
      "rewards/format_reward": 0.4375000149011612,
      "step": 55,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.04737149551510811,
      "w_max": 1.5634404420852661,
      "w_mean": 1.1619892120361328,
      "w_min": 0.0,
      "w_std": 0.2019548863172531
    },
    {
      "completion_length": 3193.3438110351562,
      "cov_mean": -6.364414912241045e-05,
      "cov_std": 0.35638032108545303,
      "entropy": 0.38623046875,
      "epoch": 0.064,
      "grad_norm": 0.22449523210525513,
      "kl": 0.000545501708984375,
      "learning_rate": 8.187213662662538e-07,
      "loss": -0.0369,
      "reward": 0.6770833432674408,
      "reward_std": 0.3087990954518318,
      "rewards/accuracy_reward": 0.2291666716337204,
      "rewards/format_reward": 0.4479166865348816,
      "step": 56,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.038341518957167864,
      "w_max": 1.6050121486186981,
      "w_mean": 1.1923900246620178,
      "w_min": 0.0,
      "w_std": 0.19538425654172897
    },
    {
      "completion_length": 3553.1563110351562,
      "cov_mean": 2.031017220360809e-05,
      "cov_std": 0.3057239428162575,
      "entropy": 0.31396484375,
      "epoch": 0.06514285714285714,
      "grad_norm": 0.1648004949092865,
      "kl": 0.0003190040588378906,
      "learning_rate": 8.102495512755938e-07,
      "loss": 0.0243,
      "reward": 0.5520833544433117,
      "reward_std": 0.47310057282447815,
      "rewards/accuracy_reward": 0.16666666977107525,
      "rewards/format_reward": 0.38541668467223644,
      "step": 57,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.04216110520064831,
      "w_max": 1.4258966147899628,
      "w_mean": 1.126460313796997,
      "w_min": 0.0,
      "w_std": 0.163984976708889
    },
    {
      "completion_length": 2358.1771240234375,
      "cov_mean": 0.00012649836571654305,
      "cov_std": 0.3942733556032181,
      "entropy": 0.32470703125,
      "epoch": 0.06628571428571428,
      "grad_norm": 0.3121250867843628,
      "kl": 0.0030040740966796875,
      "learning_rate": 8.01636806561836e-07,
      "loss": -0.0315,
      "reward": 0.9791667014360428,
      "reward_std": 0.47231587767601013,
      "rewards/accuracy_reward": 0.3020833395421505,
      "rewards/format_reward": 0.6770833358168602,
      "step": 58,
      "w_high_ratio": 0.08487696945667267,
      "w_low_ratio": 0.045225437730550766,
      "w_max": 1.9617216885089874,
      "w_mean": 1.3747560679912567,
      "w_min": 3.898036555555448e-35,
      "w_std": 0.2643117532134056
    },
    {
      "completion_length": 3183.8021850585938,
      "cov_mean": -4.030091076856479e-06,
      "cov_std": 0.21410225331783295,
      "entropy": 0.345703125,
      "epoch": 0.06742857142857143,
      "grad_norm": 0.37277111411094666,
      "kl": 0.0005140304565429688,
      "learning_rate": 7.928877960781808e-07,
      "loss": -0.0033,
      "reward": 0.5104166977107525,
      "reward_std": 0.2982303276658058,
      "rewards/accuracy_reward": 0.17708333861082792,
      "rewards/format_reward": 0.3333333432674408,
      "step": 59,
      "w_high_ratio": 0.1216658167541027,
      "w_low_ratio": 0.023135079303756356,
      "w_max": 1.7003150880336761,
      "w_mean": 1.2053856253623962,
      "w_min": 0.25,
      "w_std": 0.15523223765194416
    },
    {
      "completion_length": 3174.5000610351562,
      "cov_mean": -1.2183483704575337e-05,
      "cov_std": 0.27795324102044106,
      "entropy": 0.3603515625,
      "epoch": 0.06857142857142857,
      "grad_norm": 0.26638808846473694,
      "kl": 0.0025773048400878906,
      "learning_rate": 7.840072575681468e-07,
      "loss": 0.0243,
      "reward": 0.5729167014360428,
      "reward_std": 0.4113100916147232,
      "rewards/accuracy_reward": 0.14583334140479565,
      "rewards/format_reward": 0.4270833507180214,
      "step": 60,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03565680282190442,
      "w_max": 1.410323053598404,
      "w_mean": 1.124219536781311,
      "w_min": 0.0,
      "w_std": 0.15482920035719872
    },
    {
      "completion_length": 3304.791748046875,
      "cov_mean": 1.00996726359881e-05,
      "cov_std": 0.37024862319231033,
      "entropy": 0.3603515625,
      "epoch": 0.06971428571428571,
      "grad_norm": 0.43320998549461365,
      "kl": 0.0008983612060546875,
      "learning_rate": 7.75e-07,
      "loss": -0.0682,
      "reward": 0.645833358168602,
      "reward_std": 0.45523863658308983,
      "rewards/accuracy_reward": 0.1875000074505806,
      "rewards/format_reward": 0.4583333432674408,
      "step": 61,
      "w_high_ratio": 0.01961296983063221,
      "w_low_ratio": 0.04602003050968051,
      "w_max": 1.6873614192008972,
      "w_mean": 1.205470085144043,
      "w_min": 0.0,
      "w_std": 0.22806508466601372
    },
    {
      "completion_length": 2739.2500610351562,
      "cov_mean": 5.323334062268259e-05,
      "cov_std": 0.4578249305486679,
      "entropy": 0.31982421875,
      "epoch": 0.07085714285714285,
      "grad_norm": 0.3162655532360077,
      "kl": 0.002288818359375,
      "learning_rate": 7.658709009626109e-07,
      "loss": 0.0286,
      "reward": 0.916666679084301,
      "reward_std": 0.6331392228603363,
      "rewards/accuracy_reward": 0.2812500111758709,
      "rewards/format_reward": 0.6354166939854622,
      "step": 62,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.05417798087000847,
      "w_max": 1.768731027841568,
      "w_mean": 1.2571848034858704,
      "w_min": 0.0,
      "w_std": 0.26054797321558
    },
    {
      "completion_length": 2615.3021850585938,
      "cov_mean": 1.778580372047145e-05,
      "cov_std": 0.4087640196084976,
      "entropy": 0.4072265625,
      "epoch": 0.072,
      "grad_norm": 0.392840176820755,
      "kl": 0.0025348663330078125,
      "learning_rate": 7.566249040241553e-07,
      "loss": -0.0491,
      "reward": 1.0104167014360428,
      "reward_std": 0.5578364282846451,
      "rewards/accuracy_reward": 0.3333333507180214,
      "rewards/format_reward": 0.677083358168602,
      "step": 63,
      "w_high_ratio": 0.008992652408778667,
      "w_low_ratio": 0.04948492627590895,
      "w_max": 1.822005033493042,
      "w_mean": 1.33000847697258,
      "w_min": 2.0318827732709848e-44,
      "w_std": 0.24196847900748253
    },
    {
      "completion_length": 3171.354248046875,
      "cov_mean": -0.00014721620300406357,
      "cov_std": 0.3767973370850086,
      "entropy": 0.3984375,
      "epoch": 0.07314285714285715,
      "grad_norm": 0.4929248094558716,
      "kl": 0.0016901493072509766,
      "learning_rate": 7.472670160550848e-07,
      "loss": -0.033,
      "reward": 0.6875000204890966,
      "reward_std": 0.5117540434002876,
      "rewards/accuracy_reward": 0.2708333460614085,
      "rewards/format_reward": 0.4166666669771075,
      "step": 64,
      "w_high_ratio": 0.14000318944454193,
      "w_low_ratio": 0.03487167996354401,
      "w_max": 1.7245290279388428,
      "w_mean": 1.3096586167812347,
      "w_min": 0.0,
      "w_std": 0.2602265626192093
    },
    {
      "completion_length": 2907.3126220703125,
      "cov_mean": 3.645536344265565e-05,
      "cov_std": 0.29790719598531723,
      "entropy": 0.3271484375,
      "epoch": 0.07428571428571429,
      "grad_norm": 0.32978716492652893,
      "kl": 0.0017614364624023438,
      "learning_rate": 7.37802304516818e-07,
      "loss": -0.0106,
      "reward": 0.6875000260770321,
      "reward_std": 0.3368534557521343,
      "rewards/accuracy_reward": 0.1666666679084301,
      "rewards/format_reward": 0.5208333395421505,
      "step": 65,
      "w_high_ratio": 0.01945015788078308,
      "w_low_ratio": 0.03530450165271759,
      "w_max": 1.5614324808120728,
      "w_mean": 1.165216714143753,
      "w_min": 0.0,
      "w_std": 0.17453981935977936
    },
    {
      "completion_length": 2260.9896087646484,
      "cov_mean": -8.368766430066898e-06,
      "cov_std": 0.21803472004830837,
      "entropy": 0.310791015625,
      "epoch": 0.07542857142857143,
      "grad_norm": 0.49300873279571533,
      "kl": 0.00205230712890625,
      "learning_rate": 7.282358947176205e-07,
      "loss": 0.0218,
      "reward": 0.8750000055879354,
      "reward_std": 0.27498848363757133,
      "rewards/accuracy_reward": 0.3645833348855376,
      "rewards/format_reward": 0.5104166669771075,
      "step": 66,
      "w_high_ratio": 0.0345294363796711,
      "w_low_ratio": 0.024254919728264213,
      "w_max": 1.7126049399375916,
      "w_mean": 1.2293311953544617,
      "w_min": 1.3662660027166966e-44,
      "w_std": 0.14490841701626778
    },
    {
      "completion_length": 3728.5833740234375,
      "cov_mean": 5.284800408844603e-06,
      "cov_std": 0.1643856093287468,
      "entropy": 0.34765625,
      "epoch": 0.07657142857142857,
      "grad_norm": 0.2334880828857422,
      "kl": 0.001039743423461914,
      "learning_rate": 7.185729670371604e-07,
      "loss": 0.0054,
      "reward": 0.1770833358168602,
      "reward_std": 0.19108106940984726,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.15625,
      "step": 67,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.02258414216339588,
      "w_max": 1.245382398366928,
      "w_mean": 1.0691203624010086,
      "w_min": 0.5,
      "w_std": 0.10235420987010002
    },
    {
      "completion_length": 2355.072982788086,
      "cov_mean": -1.9176595742464997e-06,
      "cov_std": 0.27589889243245125,
      "entropy": 0.3740234375,
      "epoch": 0.07771428571428571,
      "grad_norm": 0.37152960896492004,
      "kl": 0.0041484832763671875,
      "learning_rate": 7.08818754121241e-07,
      "loss": -0.0423,
      "reward": 0.8229167014360428,
      "reward_std": 0.415886752307415,
      "rewards/accuracy_reward": 0.2291666753590107,
      "rewards/format_reward": 0.5937500074505806,
      "step": 68,
      "w_high_ratio": 0.012929616495966911,
      "w_low_ratio": 0.031258232425898314,
      "w_max": 1.8356628715991974,
      "w_mean": 1.304297924041748,
      "w_min": 0.25,
      "w_std": 0.19349467381834984
    },
    {
      "completion_length": 2847.2604370117188,
      "cov_mean": -4.8151488954317756e-05,
      "cov_std": 0.28809408843517303,
      "entropy": 0.4375,
      "epoch": 0.07885714285714286,
      "grad_norm": 0.3872397840023041,
      "kl": 0.0045623779296875,
      "learning_rate": 6.989785380482312e-07,
      "loss": -0.0566,
      "reward": 0.4479166939854622,
      "reward_std": 0.33732588589191437,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.4166666865348816,
      "step": 69,
      "w_high_ratio": 0.19539788458496332,
      "w_low_ratio": 0.03220258606597781,
      "w_max": 2.5382341742515564,
      "w_mean": 1.4389045536518097,
      "w_min": 0.0,
      "w_std": 0.1934449914842844
    },
    {
      "completion_length": 3250.4063110351562,
      "cov_mean": 4.1365366996615194e-05,
      "cov_std": 0.29582661017775536,
      "entropy": 0.34326171875,
      "epoch": 0.08,
      "grad_norm": 0.23539891839027405,
      "kl": 0.0022635459899902344,
      "learning_rate": 6.890576474687263e-07,
      "loss": 0.0138,
      "reward": 0.6145833684131503,
      "reward_std": 0.3936128318309784,
      "rewards/accuracy_reward": 0.1562500037252903,
      "rewards/format_reward": 0.4583333535119891,
      "step": 70,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03546257223933935,
      "w_max": 1.3784515261650085,
      "w_mean": 1.1171683073043823,
      "w_min": 0.0,
      "w_std": 0.1633461881428957
    },
    {
      "completion_length": 2797.4063415527344,
      "cov_mean": 4.543551585811656e-05,
      "cov_std": 0.2623286135494709,
      "entropy": 0.37744140625,
      "epoch": 0.08114285714285714,
      "grad_norm": 1.3858193159103394,
      "kl": 0.05230998992919922,
      "learning_rate": 6.790614547199906e-07,
      "loss": 0.0548,
      "reward": 0.5729166772216558,
      "reward_std": 0.24960162490606308,
      "rewards/accuracy_reward": 0.1770833358168602,
      "rewards/format_reward": 0.39583333395421505,
      "step": 71,
      "w_high_ratio": 0.13244600966572762,
      "w_low_ratio": 0.025677886325865984,
      "w_max": 1.9256412386894226,
      "w_mean": 1.2497790455818176,
      "w_min": 0.0,
      "w_std": 0.19411796145141125
    },
    {
      "completion_length": 3362.5313720703125,
      "cov_mean": 3.213349305042357e-05,
      "cov_std": 0.4226767495274544,
      "entropy": 0.486328125,
      "epoch": 0.08228571428571428,
      "grad_norm": 0.4099564254283905,
      "kl": 0.004467964172363281,
      "learning_rate": 6.68995372916741e-07,
      "loss": -0.0646,
      "reward": 0.416666679084301,
      "reward_std": 0.4769069701433182,
      "rewards/accuracy_reward": 0.0520833358168602,
      "rewards/format_reward": 0.3645833432674408,
      "step": 72,
      "w_high_ratio": 0.08334781229496002,
      "w_low_ratio": 0.05627091834321618,
      "w_max": 1.7420941889286041,
      "w_mean": 1.216987669467926,
      "w_min": 0.0,
      "w_std": 0.27180150151252747
    },
    {
      "completion_length": 3799.3334350585938,
      "cov_mean": 6.445545591304835e-05,
      "cov_std": 0.38433366641402245,
      "entropy": 0.45068359375,
      "epoch": 0.08342857142857144,
      "grad_norm": 0.2338305115699768,
      "kl": 0.00043392181396484375,
      "learning_rate": 6.588648530198504e-07,
      "loss": 0.0467,
      "reward": 0.4062500027939677,
      "reward_std": 0.48472320288419724,
      "rewards/accuracy_reward": 0.1770833395421505,
      "rewards/format_reward": 0.22916667442768812,
      "step": 73,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.054560547694563866,
      "w_max": 1.3258715867996216,
      "w_mean": 1.0365844666957855,
      "w_min": 2.0397733447937514e-38,
      "w_std": 0.1829804591834545
    },
    {
      "completion_length": 3411.354248046875,
      "cov_mean": 2.615013909235131e-05,
      "cov_std": 0.32939745485782623,
      "entropy": 0.38916015625,
      "epoch": 0.08457142857142858,
      "grad_norm": 0.31615766882896423,
      "kl": 0.0020766258239746094,
      "learning_rate": 6.486753808845564e-07,
      "loss": 0.0447,
      "reward": 0.5416666716337204,
      "reward_std": 0.48268113285303116,
      "rewards/accuracy_reward": 0.2500000037252903,
      "rewards/format_reward": 0.2916666716337204,
      "step": 74,
      "w_high_ratio": 0.017251405864953995,
      "w_low_ratio": 0.037632704712450504,
      "w_max": 1.4793908894062042,
      "w_mean": 1.120386153459549,
      "w_min": 0.25,
      "w_std": 0.18761293590068817
    },
    {
      "completion_length": 3256.6876220703125,
      "cov_mean": 7.561668553535128e-07,
      "cov_std": 0.39095795527100563,
      "entropy": 0.36669921875,
      "epoch": 0.08571428571428572,
      "grad_norm": 0.29881325364112854,
      "kl": 0.0019350051879882812,
      "learning_rate": 6.384324742897735e-07,
      "loss": 0.0709,
      "reward": 0.5625000260770321,
      "reward_std": 0.3942164406180382,
      "rewards/accuracy_reward": 0.1979166716337204,
      "rewards/format_reward": 0.36458334513008595,
      "step": 75,
      "w_high_ratio": 0.05663827061653137,
      "w_low_ratio": 0.04413987882435322,
      "w_max": 1.78597491979599,
      "w_mean": 1.1896328330039978,
      "w_min": 0.0,
      "w_std": 0.20989646948873997
    },
    {
      "completion_length": 3081.6875610351562,
      "cov_mean": 7.851473583286861e-06,
      "cov_std": 0.19348382577300072,
      "entropy": 0.38671875,
      "epoch": 0.08685714285714285,
      "grad_norm": 0.13885051012039185,
      "kl": 0.0005955696105957031,
      "learning_rate": 6.281416799501187e-07,
      "loss": -0.0033,
      "reward": 0.4583333432674408,
      "reward_std": 0.22215576469898224,
      "rewards/accuracy_reward": 0.0520833358168602,
      "rewards/format_reward": 0.4062500074505806,
      "step": 76,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.024444932583719492,
      "w_max": 1.385847419500351,
      "w_mean": 1.104325294494629,
      "w_min": 0.25,
      "w_std": 0.09913814999163151
    },
    {
      "completion_length": 3410.2084350585938,
      "cov_mean": 1.7174193999380805e-05,
      "cov_std": 0.28355711698532104,
      "entropy": 0.41064453125,
      "epoch": 0.088,
      "grad_norm": 0.220694899559021,
      "kl": 0.0006546974182128906,
      "learning_rate": 6.178085705122674e-07,
      "loss": -0.0111,
      "reward": 0.46875000558793545,
      "reward_std": 0.2968830242753029,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.34375000558793545,
      "step": 77,
      "w_high_ratio": 0.03967808559536934,
      "w_low_ratio": 0.03201043838635087,
      "w_max": 1.494775265455246,
      "w_mean": 1.1476246118545532,
      "w_min": 0.25,
      "w_std": 0.1502416580915451
    },
    {
      "completion_length": 3454.6771850585938,
      "cov_mean": 2.8414152438926976e-05,
      "cov_std": 0.42589128017425537,
      "entropy": 0.376953125,
      "epoch": 0.08914285714285715,
      "grad_norm": 0.24352695047855377,
      "kl": 0.0004711151123046875,
      "learning_rate": 6.074387415372676e-07,
      "loss": -0.0208,
      "reward": 0.6354166716337204,
      "reward_std": 0.5373464152216911,
      "rewards/accuracy_reward": 0.2500000111758709,
      "rewards/format_reward": 0.3854166716337204,
      "step": 78,
      "w_high_ratio": 0.040874604135751724,
      "w_low_ratio": 0.04665249306708574,
      "w_max": 1.7644164860248566,
      "w_mean": 1.1853148639202118,
      "w_min": 1.401298464324817e-45,
      "w_std": 0.24474802613258362
    },
    {
      "completion_length": 2533.354217529297,
      "cov_mean": 4.250231540936511e-06,
      "cov_std": 0.28236184269189835,
      "entropy": 0.322265625,
      "epoch": 0.09028571428571429,
      "grad_norm": 0.2319362461566925,
      "kl": 0.0014781951904296875,
      "learning_rate": 5.97037808470444e-07,
      "loss": 0.0229,
      "reward": 0.8541666865348816,
      "reward_std": 0.3287995010614395,
      "rewards/accuracy_reward": 0.23958333861082792,
      "rewards/format_reward": 0.6145833507180214,
      "step": 79,
      "w_high_ratio": 0.16244123131036758,
      "w_low_ratio": 0.035402802750468254,
      "w_max": 2.477913051843643,
      "w_mean": 1.4660456776618958,
      "w_min": 0.3189111649990082,
      "w_std": 0.194712957367301
    },
    {
      "completion_length": 3480.1250610351562,
      "cov_mean": -2.1006295810366282e-05,
      "cov_std": 0.1846322864294052,
      "entropy": 0.4541015625,
      "epoch": 0.09142857142857143,
      "grad_norm": 0.3083387017250061,
      "kl": 0.0014166831970214844,
      "learning_rate": 5.866114036005362e-07,
      "loss": 0.0178,
      "reward": 0.6041666893288493,
      "reward_std": 0.32849549502134323,
      "rewards/accuracy_reward": 0.2187500111758709,
      "rewards/format_reward": 0.3854166781529784,
      "step": 80,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.023656398989260197,
      "w_max": 1.5105722546577454,
      "w_mean": 1.109735906124115,
      "w_min": 0.25,
      "w_std": 0.10967518202960491
    },
    {
      "completion_length": 3358.6979370117188,
      "cov_mean": 1.4671212284156354e-05,
      "cov_std": 0.1834326833486557,
      "entropy": 0.5087890625,
      "epoch": 0.09257142857142857,
      "grad_norm": 0.17795835435390472,
      "kl": 0.005743980407714844,
      "learning_rate": 5.761651730097142e-07,
      "loss": -0.0007,
      "reward": 0.36458334885537624,
      "reward_std": 0.28906675428152084,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.2708333386108279,
      "step": 81,
      "w_high_ratio": 0.0625,
      "w_low_ratio": 0.025394567288458347,
      "w_max": 1.5137894749641418,
      "w_mean": 1.1780100166797638,
      "w_min": 0.25,
      "w_std": 0.09445377439260483
    },
    {
      "completion_length": 3041.3438110351562,
      "cov_mean": -2.3124930976337055e-05,
      "cov_std": 0.2670583054423332,
      "entropy": 0.39501953125,
      "epoch": 0.09371428571428571,
      "grad_norm": 0.30143725872039795,
      "kl": 0.0016946792602539062,
      "learning_rate": 5.657047735161255e-07,
      "loss": -0.0214,
      "reward": 0.7604166716337204,
      "reward_std": 0.34568001329898834,
      "rewards/accuracy_reward": 0.3125000074505806,
      "rewards/format_reward": 0.4479166716337204,
      "step": 82,
      "w_high_ratio": 0.04279303178191185,
      "w_low_ratio": 0.03479792643338442,
      "w_max": 1.7097257375717163,
      "w_mean": 1.1838775873184204,
      "w_min": 0.25,
      "w_std": 0.16872986778616905
    },
    {
      "completion_length": 3054.3646240234375,
      "cov_mean": -5.689870704372879e-06,
      "cov_std": 0.17481233924627304,
      "entropy": 0.43115234375,
      "epoch": 0.09485714285714286,
      "grad_norm": 0.4099189341068268,
      "kl": 0.0016026496887207031,
      "learning_rate": 5.552358696106288e-07,
      "loss": 0.0418,
      "reward": 0.46875,
      "reward_std": 0.2636485621333122,
      "rewards/accuracy_reward": 0.16666666697710752,
      "rewards/format_reward": 0.30208333395421505,
      "step": 83,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.01848737057298422,
      "w_max": 1.5311354398727417,
      "w_mean": 1.116348922252655,
      "w_min": 0.5,
      "w_std": 0.08476846665143967
    },
    {
      "completion_length": 3265.4375,
      "cov_mean": 2.6241104933433235e-06,
      "cov_std": 0.3729252219200134,
      "entropy": 0.40966796875,
      "epoch": 0.096,
      "grad_norm": 0.3238582909107208,
      "kl": 0.0008416175842285156,
      "learning_rate": 5.447641303893714e-07,
      "loss": 0.0273,
      "reward": 0.6770833432674408,
      "reward_std": 0.4903585724532604,
      "rewards/accuracy_reward": 0.2500000074505806,
      "rewards/format_reward": 0.4270833432674408,
      "step": 84,
      "w_high_ratio": 0.01827232539653778,
      "w_low_ratio": 0.04356031212955713,
      "w_max": 1.6150497496128082,
      "w_mean": 1.17939093708992,
      "w_min": 0.0,
      "w_std": 0.2226697877049446
    },
    {
      "completion_length": 3305.8125,
      "cov_mean": 5.61498741262767e-05,
      "cov_std": 0.4452364891767502,
      "entropy": 0.32861328125,
      "epoch": 0.09714285714285714,
      "grad_norm": 0.29220932722091675,
      "kl": 0.0005831718444824219,
      "learning_rate": 5.342952264838747e-07,
      "loss": 0.0609,
      "reward": 0.6354166865348816,
      "reward_std": 0.68822330981493,
      "rewards/accuracy_reward": 0.1875,
      "rewards/format_reward": 0.447916679084301,
      "step": 85,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.046569294296205044,
      "w_max": 1.4473570883274078,
      "w_mean": 1.1975693106651306,
      "w_min": 0.0,
      "w_std": 0.23973201215267181
    },
    {
      "completion_length": 3247.2188110351562,
      "cov_mean": -1.9998861716885585e-05,
      "cov_std": 0.31147949025034904,
      "entropy": 0.44189453125,
      "epoch": 0.09828571428571428,
      "grad_norm": 0.5364130735397339,
      "kl": 0.0019941329956054688,
      "learning_rate": 5.238348269902859e-07,
      "loss": -0.0757,
      "reward": 0.6041666865348816,
      "reward_std": 0.28646790236234665,
      "rewards/accuracy_reward": 0.1979166716337204,
      "rewards/format_reward": 0.40625,
      "step": 86,
      "w_high_ratio": 0.1384273413568735,
      "w_low_ratio": 0.027202811557799578,
      "w_max": 2.1997461020946503,
      "w_mean": 1.3846492767333984,
      "w_min": 0.25,
      "w_std": 0.2737709581851959
    },
    {
      "completion_length": 3057.2916870117188,
      "cov_mean": -5.754891753895208e-05,
      "cov_std": 0.5682927817106247,
      "entropy": 0.4931640625,
      "epoch": 0.09942857142857142,
      "grad_norm": 0.587156355381012,
      "kl": 0.0027523040771484375,
      "learning_rate": 5.133885963994639e-07,
      "loss": -0.0103,
      "reward": 0.7083333432674408,
      "reward_std": 0.560588151216507,
      "rewards/accuracy_reward": 0.21875000279396772,
      "rewards/format_reward": 0.4895833432674408,
      "step": 87,
      "w_high_ratio": 0.15174898132681847,
      "w_low_ratio": 0.06858384050428867,
      "w_max": 2.0574756860733032,
      "w_mean": 1.3842451870441437,
      "w_min": 0.0,
      "w_std": 0.38674013316631317
    },
    {
      "completion_length": 2906.0209350585938,
      "cov_mean": 4.286354305804707e-05,
      "cov_std": 0.5042391121387482,
      "entropy": 0.4453125,
      "epoch": 0.10057142857142858,
      "grad_norm": 0.44004324078559875,
      "kl": 0.0045490264892578125,
      "learning_rate": 5.02962191529556e-07,
      "loss": 0.0208,
      "reward": 0.9791666865348816,
      "reward_std": 0.5309341698884964,
      "rewards/accuracy_reward": 0.3437500074505806,
      "rewards/format_reward": 0.6354166716337204,
      "step": 88,
      "w_high_ratio": 0.17751162499189377,
      "w_low_ratio": 0.05830758810043335,
      "w_max": 2.042703092098236,
      "w_mean": 1.399275004863739,
      "w_min": 0.25,
      "w_std": 0.35736314207315445
    },
    {
      "completion_length": 3425.3333740234375,
      "cov_mean": -2.9065696480756742e-05,
      "cov_std": 0.3547215014696121,
      "entropy": 0.39208984375,
      "epoch": 0.10171428571428572,
      "grad_norm": 0.41978099942207336,
      "kl": 0.0021228790283203125,
      "learning_rate": 4.925612584627324e-07,
      "loss": -0.0597,
      "reward": 0.5625000223517418,
      "reward_std": 0.5216004773974419,
      "rewards/accuracy_reward": 0.2083333358168602,
      "rewards/format_reward": 0.354166679084301,
      "step": 89,
      "w_high_ratio": 0.017988620325922966,
      "w_low_ratio": 0.042721704579889774,
      "w_max": 1.6596693396568298,
      "w_mean": 1.1673058569431305,
      "w_min": 0.0,
      "w_std": 0.22802076116204262
    },
    {
      "completion_length": 2660.4375915527344,
      "cov_mean": 9.994783340516733e-06,
      "cov_std": 0.2509019151329994,
      "entropy": 0.546875,
      "epoch": 0.10285714285714286,
      "grad_norm": 0.7049030661582947,
      "kl": 0.0055828094482421875,
      "learning_rate": 4.821914294877326e-07,
      "loss": -0.0314,
      "reward": 0.48958335630595684,
      "reward_std": 0.24196770787239075,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.4687500176951289,
      "step": 90,
      "w_high_ratio": 0.09902577847242355,
      "w_low_ratio": 0.025217004818841815,
      "w_max": 2.4860771000385284,
      "w_mean": 1.4202724397182465,
      "w_min": 0.0,
      "w_std": 0.18205549381673336
    },
    {
      "completion_length": 3260.8334350585938,
      "cov_mean": -1.9221572074457072e-05,
      "cov_std": 0.2735915407538414,
      "entropy": 0.41845703125,
      "epoch": 0.104,
      "grad_norm": 0.2550157308578491,
      "kl": 0.0021266937255859375,
      "learning_rate": 4.7185832004988133e-07,
      "loss": 0.0421,
      "reward": 0.5312500316649675,
      "reward_std": 0.3842952623963356,
      "rewards/accuracy_reward": 0.14583333674818277,
      "rewards/format_reward": 0.385416672565043,
      "step": 91,
      "w_high_ratio": 0.042984794825315475,
      "w_low_ratio": 0.027249778620898724,
      "w_max": 1.7297326922416687,
      "w_mean": 1.2372649610042572,
      "w_min": 0.0,
      "w_std": 0.1690264195203781
    },
    {
      "completion_length": 2809.573028564453,
      "cov_mean": -4.66689198219683e-06,
      "cov_std": 0.41415752843022346,
      "entropy": 0.34033203125,
      "epoch": 0.10514285714285715,
      "grad_norm": 0.3232761323451996,
      "kl": 0.0012707710266113281,
      "learning_rate": 4.6156752571022637e-07,
      "loss": -0.0264,
      "reward": 0.7083333432674408,
      "reward_std": 0.48402564972639084,
      "rewards/accuracy_reward": 0.16666667722165585,
      "rewards/format_reward": 0.5416666939854622,
      "step": 92,
      "w_high_ratio": 0.0625,
      "w_low_ratio": 0.04879668727517128,
      "w_max": 1.7188981473445892,
      "w_mean": 1.264578402042389,
      "w_min": 1.0121512603752466e-34,
      "w_std": 0.23184099607169628
    },
    {
      "completion_length": 3819.1458740234375,
      "cov_mean": -4.416617707647674e-05,
      "cov_std": 0.18136212974786758,
      "entropy": 0.533203125,
      "epoch": 0.10628571428571429,
      "grad_norm": 0.2244294136762619,
      "kl": 0.002910614013671875,
      "learning_rate": 4.513246191154434e-07,
      "loss": 0.0303,
      "reward": 0.0937500037252903,
      "reward_std": 0.1851910501718521,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.08333333674818277,
      "step": 93,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.025186134036630392,
      "w_max": 1.3321870565414429,
      "w_mean": 1.0297031998634338,
      "w_min": 0.25,
      "w_std": 0.09247609600424767
    },
    {
      "completion_length": 3216.4376220703125,
      "cov_mean": -2.7046048671763856e-05,
      "cov_std": 0.25123433768749237,
      "entropy": 0.5078125,
      "epoch": 0.10742857142857143,
      "grad_norm": 0.2843908965587616,
      "kl": 0.0036516189575195312,
      "learning_rate": 4.4113514698014953e-07,
      "loss": 0.0352,
      "reward": 0.47916667722165585,
      "reward_std": 0.28607168793678284,
      "rewards/accuracy_reward": 0.17708333395421505,
      "rewards/format_reward": 0.30208334140479565,
      "step": 94,
      "w_high_ratio": 0.04015461727976799,
      "w_low_ratio": 0.02874834556132555,
      "w_max": 1.7613461911678314,
      "w_mean": 1.1813118755817413,
      "w_min": 0.25,
      "w_std": 0.16030845791101456
    },
    {
      "completion_length": 3702.510498046875,
      "cov_mean": 3.147694224026054e-05,
      "cov_std": 0.3758469521999359,
      "entropy": 0.4091796875,
      "epoch": 0.10857142857142857,
      "grad_norm": 0.2823449671268463,
      "kl": 0.0005347728729248047,
      "learning_rate": 4.3100462708325914e-07,
      "loss": 0.0264,
      "reward": 0.416666679084301,
      "reward_std": 0.5112503468990326,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.3020833432674408,
      "step": 95,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.0470161447301507,
      "w_max": 1.2909899652004242,
      "w_mean": 1.0598113238811493,
      "w_min": 0.25,
      "w_std": 0.19865867495536804
    },
    {
      "completion_length": 3136.7188110351562,
      "cov_mean": 2.809584930218989e-05,
      "cov_std": 0.38921112939715385,
      "entropy": 0.41259765625,
      "epoch": 0.10971428571428571,
      "grad_norm": 0.4165164828300476,
      "kl": 0.004611968994140625,
      "learning_rate": 4.209385452800095e-07,
      "loss": -0.0197,
      "reward": 0.677083358168602,
      "reward_std": 0.4686770662665367,
      "rewards/accuracy_reward": 0.25000000558793545,
      "rewards/format_reward": 0.4270833432674408,
      "step": 96,
      "w_high_ratio": 0.1767394095659256,
      "w_low_ratio": 0.046211169101297855,
      "w_max": 2.1192705631256104,
      "w_mean": 1.334891527891159,
      "w_min": 2.0274371819188836e-36,
      "w_std": 0.27778077498078346
    },
    {
      "completion_length": 3338.3958740234375,
      "cov_mean": 4.9627053158474155e-05,
      "cov_std": 0.3888591527938843,
      "entropy": 0.41650390625,
      "epoch": 0.11085714285714286,
      "grad_norm": 0.41793736815452576,
      "kl": 0.0014972686767578125,
      "learning_rate": 4.1094235253127374e-07,
      "loss": -0.0162,
      "reward": 0.5520833563059568,
      "reward_std": 0.5027666687965393,
      "rewards/accuracy_reward": 0.20833334419876337,
      "rewards/format_reward": 0.3437500176951289,
      "step": 97,
      "w_high_ratio": 0.036094631999731064,
      "w_low_ratio": 0.03847068129107356,
      "w_max": 1.7370452582836151,
      "w_mean": 1.2170022130012512,
      "w_min": 0.0,
      "w_std": 0.2471493650227785
    },
    {
      "completion_length": 3080.9584350585938,
      "cov_mean": -3.7720649288530694e-05,
      "cov_std": 0.35732631012797356,
      "entropy": 0.37109375,
      "epoch": 0.112,
      "grad_norm": 0.4429682493209839,
      "kl": 0.0007581710815429688,
      "learning_rate": 4.0102146195176887e-07,
      "loss": -0.087,
      "reward": 0.614583358168602,
      "reward_std": 0.3441091701388359,
      "rewards/accuracy_reward": 0.15625000558793545,
      "rewards/format_reward": 0.4583333507180214,
      "step": 98,
      "w_high_ratio": 0.07888161391019821,
      "w_low_ratio": 0.0333328228443861,
      "w_max": 1.875957041978836,
      "w_mean": 1.2836172580718994,
      "w_min": 0.0,
      "w_std": 0.24231520667672157
    },
    {
      "completion_length": 3051.885467529297,
      "cov_mean": -8.294958661281271e-06,
      "cov_std": 0.13039003312587738,
      "entropy": 0.342529296875,
      "epoch": 0.11314285714285714,
      "grad_norm": 0.09671887010335922,
      "kl": 0.0012235641479492188,
      "learning_rate": 3.911812458787591e-07,
      "loss": 0.015,
      "reward": 0.5520833395421505,
      "reward_std": 0.22183798253536224,
      "rewards/accuracy_reward": 0.20833333395421505,
      "rewards/format_reward": 0.34375000558793545,
      "step": 99,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.016014322638511658,
      "w_max": 1.2210949659347534,
      "w_mean": 1.0700498223304749,
      "w_min": 0.5234909653663635,
      "w_std": 0.06671209260821342
    },
    {
      "completion_length": 2941.4271850585938,
      "cov_mean": 3.641827424871735e-05,
      "cov_std": 0.311247356235981,
      "entropy": 0.352783203125,
      "epoch": 0.11428571428571428,
      "grad_norm": 0.40575236082077026,
      "kl": 0.0022411346435546875,
      "learning_rate": 3.8142703296283953e-07,
      "loss": -0.0488,
      "reward": 0.8229167088866234,
      "reward_std": 0.4384620487689972,
      "rewards/accuracy_reward": 0.3020833507180214,
      "rewards/format_reward": 0.5208333507180214,
      "step": 100,
      "w_high_ratio": 0.057203881442546844,
      "w_low_ratio": 0.037438319995999336,
      "w_max": 2.0794378519058228,
      "w_mean": 1.3217148184776306,
      "w_min": 0.0,
      "w_std": 0.19731487706303596
    },
    {
      "completion_length": 3064.322967529297,
      "cov_mean": -0.00016853955486340055,
      "cov_std": 0.3906340226531029,
      "entropy": 0.49560546875,
      "epoch": 0.11542857142857142,
      "grad_norm": 0.3836102783679962,
      "kl": 0.0015306472778320312,
      "learning_rate": 3.7176410528237945e-07,
      "loss": 0.0071,
      "reward": 0.6041666716337204,
      "reward_std": 0.37709444761276245,
      "rewards/accuracy_reward": 0.20833334419876337,
      "rewards/format_reward": 0.3958333469927311,
      "step": 101,
      "w_high_ratio": 0.039534129202365875,
      "w_low_ratio": 0.04537520185112953,
      "w_max": 1.8732931017875671,
      "w_mean": 1.2176434397697449,
      "w_min": 1.2642545573704715e-38,
      "w_std": 0.2533961348235607
    },
    {
      "completion_length": 2447.7188415527344,
      "cov_mean": 6.931000660870268e-05,
      "cov_std": 0.49404649436473846,
      "entropy": 0.41943359375,
      "epoch": 0.11657142857142858,
      "grad_norm": 0.8746929168701172,
      "kl": 0.0036649703979492188,
      "learning_rate": 3.62197695483182e-07,
      "loss": -0.1189,
      "reward": 0.8541666939854622,
      "reward_std": 0.36090725660324097,
      "rewards/accuracy_reward": 0.15625000279396772,
      "rewards/format_reward": 0.6979166865348816,
      "step": 102,
      "w_high_ratio": 0.21037982031702995,
      "w_low_ratio": 0.04308201279491186,
      "w_max": 3.2181393206119537,
      "w_mean": 1.75843146443367,
      "w_min": 0.0,
      "w_std": 0.38950372859835625
    },
    {
      "completion_length": 2925.8646850585938,
      "cov_mean": 7.4649460657383315e-06,
      "cov_std": 0.41748112440109253,
      "entropy": 0.384765625,
      "epoch": 0.11771428571428572,
      "grad_norm": 0.6049470901489258,
      "kl": 0.0027923583984375,
      "learning_rate": 3.5273298394491515e-07,
      "loss": -0.0624,
      "reward": 0.8125000298023224,
      "reward_std": 0.4502665400505066,
      "rewards/accuracy_reward": 0.21875,
      "rewards/format_reward": 0.5937500149011612,
      "step": 103,
      "w_high_ratio": 0.21904528141021729,
      "w_low_ratio": 0.04505133908241987,
      "w_max": 2.339956372976303,
      "w_mean": 1.4152106940746307,
      "w_min": 0.0,
      "w_std": 0.29690178483724594
    },
    {
      "completion_length": 3003.3230590820312,
      "cov_mean": -3.357986315677408e-07,
      "cov_std": 0.30858776718378067,
      "entropy": 0.4453125,
      "epoch": 0.11885714285714286,
      "grad_norm": 1.4959157705307007,
      "kl": 0.016025543212890625,
      "learning_rate": 3.433750959758446e-07,
      "loss": -0.0262,
      "reward": 0.5833333432674408,
      "reward_std": 0.418441042304039,
      "rewards/accuracy_reward": 0.1666666716337204,
      "rewards/format_reward": 0.4166666828095913,
      "step": 104,
      "w_high_ratio": 0.11394603550434113,
      "w_low_ratio": 0.0399134736508131,
      "w_max": 1.7974587678909302,
      "w_mean": 1.2668271660804749,
      "w_min": 0.25,
      "w_std": 0.18237562477588654
    },
    {
      "completion_length": 2974.947998046875,
      "cov_mean": -6.136376214271877e-06,
      "cov_std": 0.4218733385205269,
      "entropy": 0.38671875,
      "epoch": 0.12,
      "grad_norm": 0.5280556678771973,
      "kl": 0.004073143005371094,
      "learning_rate": 3.3412909903738936e-07,
      "loss": -0.0267,
      "reward": 0.8020833535119891,
      "reward_std": 0.6002020314335823,
      "rewards/accuracy_reward": 0.322916679084301,
      "rewards/format_reward": 0.4791666818782687,
      "step": 105,
      "w_high_ratio": 0.07510412856936455,
      "w_low_ratio": 0.05238847387954593,
      "w_max": 2.132063180208206,
      "w_mean": 1.2634376883506775,
      "w_min": 0.0,
      "w_std": 0.2662976738065481
    },
    {
      "completion_length": 2262.4375915527344,
      "cov_mean": 7.913076842669398e-07,
      "cov_std": 0.2457173652946949,
      "entropy": 0.291748046875,
      "epoch": 0.12114285714285715,
      "grad_norm": 0.2537207007408142,
      "kl": 0.00292205810546875,
      "learning_rate": 3.250000000000001e-07,
      "loss": 0.0557,
      "reward": 1.0833333535119891,
      "reward_std": 0.31829095631837845,
      "rewards/accuracy_reward": 0.4270833395421505,
      "rewards/format_reward": 0.6562500251457095,
      "step": 106,
      "w_high_ratio": 0.05819880962371826,
      "w_low_ratio": 0.029597220942378044,
      "w_max": 2.0277227461338043,
      "w_mean": 1.3821330666542053,
      "w_min": 0.0,
      "w_std": 0.1694270297884941
    },
    {
      "completion_length": 3115.3021240234375,
      "cov_mean": -5.469087955134455e-05,
      "cov_std": 0.3848882205784321,
      "entropy": 0.5244140625,
      "epoch": 0.12228571428571429,
      "grad_norm": 0.38348808884620667,
      "kl": 0.0030164718627929688,
      "learning_rate": 3.159927424318531e-07,
      "loss": -0.0623,
      "reward": 0.6250000251457095,
      "reward_std": 0.3455836847424507,
      "rewards/accuracy_reward": 0.1666666716337204,
      "rewards/format_reward": 0.4583333535119891,
      "step": 107,
      "w_high_ratio": 0.16982319951057434,
      "w_low_ratio": 0.04572468576952815,
      "w_max": 1.996431291103363,
      "w_mean": 1.4110250174999237,
      "w_min": 0.0,
      "w_std": 0.2217676378786564
    },
    {
      "completion_length": 3326.885498046875,
      "cov_mean": -3.166737906212802e-05,
      "cov_std": 0.34976962953805923,
      "entropy": 0.42529296875,
      "epoch": 0.12342857142857143,
      "grad_norm": 0.4025486409664154,
      "kl": 0.0019197463989257812,
      "learning_rate": 3.0711220392181934e-07,
      "loss": 0.0339,
      "reward": 0.5104166716337204,
      "reward_std": 0.45747723430395126,
      "rewards/accuracy_reward": 0.1875000074505806,
      "rewards/format_reward": 0.3229166716337204,
      "step": 108,
      "w_high_ratio": 0.05910671502351761,
      "w_low_ratio": 0.04742087051272392,
      "w_max": 1.6517033874988556,
      "w_mean": 1.2254261672496796,
      "w_min": 0.25,
      "w_std": 0.23562095686793327
    },
    {
      "completion_length": 3175.3541870117188,
      "cov_mean": 7.027760148048401e-06,
      "cov_std": 0.17652258835732937,
      "entropy": 0.388671875,
      "epoch": 0.12457142857142857,
      "grad_norm": 0.47681912779808044,
      "kl": 0.0006990432739257812,
      "learning_rate": 2.9836319343816397e-07,
      "loss": -0.0293,
      "reward": 0.4895833460614085,
      "reward_std": 0.20087094604969025,
      "rewards/accuracy_reward": 0.12500000279396772,
      "rewards/format_reward": 0.3645833386108279,
      "step": 109,
      "w_high_ratio": 0.11448103934526443,
      "w_low_ratio": 0.018031115527264774,
      "w_max": 1.5564889311790466,
      "w_mean": 1.2944257855415344,
      "w_min": 0.25,
      "w_std": 0.10801565833389759
    },
    {
      "completion_length": 2903.9375610351562,
      "cov_mean": 2.2163485482451506e-05,
      "cov_std": 0.3046695999801159,
      "entropy": 0.37548828125,
      "epoch": 0.12571428571428572,
      "grad_norm": 0.574763298034668,
      "kl": 0.0011281967163085938,
      "learning_rate": 2.897504487244061e-07,
      "loss": 0.0666,
      "reward": 0.6458333656191826,
      "reward_std": 0.4784049317240715,
      "rewards/accuracy_reward": 0.16666667256504297,
      "rewards/format_reward": 0.479166679084301,
      "step": 110,
      "w_high_ratio": 0.11397865414619446,
      "w_low_ratio": 0.036777073866687715,
      "w_max": 1.7808756828308105,
      "w_mean": 1.2882322669029236,
      "w_min": 0.0,
      "w_std": 0.17456290312111378
    },
    {
      "completion_length": 3608.7813110351562,
      "cov_mean": 2.5331736196676502e-05,
      "cov_std": 0.3403046578168869,
      "entropy": 0.54296875,
      "epoch": 0.12685714285714286,
      "grad_norm": 0.28652071952819824,
      "kl": 0.0020961761474609375,
      "learning_rate": 2.812786337337463e-07,
      "loss": -0.0593,
      "reward": 0.4479166716337204,
      "reward_std": 0.5819729715585709,
      "rewards/accuracy_reward": 0.19791667442768812,
      "rewards/format_reward": 0.25000000558793545,
      "step": 111,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.040594917722046375,
      "w_max": 1.5843260884284973,
      "w_mean": 1.1159851551055908,
      "w_min": 0.0,
      "w_std": 0.1932711023837328
    },
    {
      "completion_length": 3511.9791870117188,
      "cov_mean": -8.262183837359771e-05,
      "cov_std": 0.3419903479516506,
      "entropy": 0.466796875,
      "epoch": 0.128,
      "grad_norm": 0.21528662741184235,
      "kl": 0.0010385513305664062,
      "learning_rate": 2.729523361034538e-07,
      "loss": 0.0036,
      "reward": 0.7395833432674408,
      "reward_std": 0.4909324310719967,
      "rewards/accuracy_reward": 0.3229166716337204,
      "rewards/format_reward": 0.4166666716337204,
      "step": 112,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.039533226285129786,
      "w_max": 1.4701470732688904,
      "w_mean": 1.1230643689632416,
      "w_min": 2.796174871268219e-27,
      "w_std": 0.17366146482527256
    },
    {
      "completion_length": 2989.6250610351562,
      "cov_mean": -9.21895634746761e-05,
      "cov_std": 0.4826783090829849,
      "entropy": 0.47412109375,
      "epoch": 0.12914285714285714,
      "grad_norm": 0.48934122920036316,
      "kl": 0.0016803741455078125,
      "learning_rate": 2.6477606467058035e-07,
      "loss": -0.0453,
      "reward": 0.739583358168602,
      "reward_std": 0.46098607778549194,
      "rewards/accuracy_reward": 0.22916666977107525,
      "rewards/format_reward": 0.510416679084301,
      "step": 113,
      "w_high_ratio": 0.05894821137189865,
      "w_low_ratio": 0.05880188010632992,
      "w_max": 2.289375811815262,
      "w_mean": 1.3520435392856598,
      "w_min": 0.0,
      "w_std": 0.3024050109088421
    },
    {
      "completion_length": 2825.6771850585938,
      "cov_mean": 8.050216638366692e-05,
      "cov_std": 0.36271025612950325,
      "entropy": 0.38232421875,
      "epoch": 0.13028571428571428,
      "grad_norm": 0.3449488580226898,
      "kl": 0.006592750549316406,
      "learning_rate": 2.567542470303452e-07,
      "loss": -0.0924,
      "reward": 0.6250000298023224,
      "reward_std": 0.24164991825819016,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.5625000149011612,
      "step": 114,
      "w_high_ratio": 0.04218151792883873,
      "w_low_ratio": 0.03556477651000023,
      "w_max": 1.9164948165416718,
      "w_mean": 1.3027912080287933,
      "w_min": 0.25,
      "w_std": 0.2133668176829815
    },
    {
      "completion_length": 3166.1146240234375,
      "cov_mean": 5.38662197868689e-06,
      "cov_std": 0.25203782320022583,
      "entropy": 0.43505859375,
      "epoch": 0.13142857142857142,
      "grad_norm": 0.26439687609672546,
      "kl": 0.0024480819702148438,
      "learning_rate": 2.488912271385139e-07,
      "loss": -0.0066,
      "reward": 0.5937500298023224,
      "reward_std": 0.28200745210051537,
      "rewards/accuracy_reward": 0.2708333358168602,
      "rewards/format_reward": 0.3229166716337204,
      "step": 115,
      "w_high_ratio": 0.10065623372793198,
      "w_low_ratio": 0.02666568197309971,
      "w_max": 1.5669940114021301,
      "w_mean": 1.224227637052536,
      "w_min": 0.25,
      "w_std": 0.1499568410217762
    },
    {
      "completion_length": 3656.2604370117188,
      "cov_mean": -1.988131225516554e-05,
      "cov_std": 0.35770974680781364,
      "entropy": 0.5078125,
      "epoch": 0.13257142857142856,
      "grad_norm": 0.29910722374916077,
      "kl": 0.0016193389892578125,
      "learning_rate": 2.411912629590699e-07,
      "loss": 0.0044,
      "reward": 0.2812500027939677,
      "reward_std": 0.3736678585410118,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.19791667442768812,
      "step": 116,
      "w_high_ratio": 0.055632032454013824,
      "w_low_ratio": 0.04639715701341629,
      "w_max": 1.5884924530982971,
      "w_mean": 1.1101520657539368,
      "w_min": 0.25,
      "w_std": 0.2023993842303753
    },
    {
      "completion_length": 3502.0833740234375,
      "cov_mean": 1.3217656032793457e-05,
      "cov_std": 0.24507246166467667,
      "entropy": 0.48486328125,
      "epoch": 0.1337142857142857,
      "grad_norm": 0.19559521973133087,
      "kl": 0.0014400482177734375,
      "learning_rate": 2.336585241584522e-07,
      "loss": 0.0266,
      "reward": 0.31250000558793545,
      "reward_std": 0.2934442162513733,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.2708333386108279,
      "step": 117,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03416412137448788,
      "w_max": 1.3934015035629272,
      "w_mean": 1.121408373117447,
      "w_min": 0.25,
      "w_std": 0.12930788472294807
    },
    {
      "completion_length": 3388.2291870117188,
      "cov_mean": 8.877824029696058e-05,
      "cov_std": 0.4343739002943039,
      "entropy": 0.39794921875,
      "epoch": 0.13485714285714287,
      "grad_norm": 0.2352270781993866,
      "kl": 0.001239776611328125,
      "learning_rate": 2.2629708984760706e-07,
      "loss": -0.001,
      "reward": 0.8750000596046448,
      "reward_std": 0.7063143625855446,
      "rewards/accuracy_reward": 0.4166666865348816,
      "rewards/format_reward": 0.458333358168602,
      "step": 118,
      "w_high_ratio": 0.025391947478055954,
      "w_low_ratio": 0.0545379314571619,
      "w_max": 1.6127934455871582,
      "w_mean": 1.1610458493232727,
      "w_min": 0.0,
      "w_std": 0.23880053497850895
    },
    {
      "completion_length": 2543.125030517578,
      "cov_mean": -5.281608531504389e-05,
      "cov_std": 0.3761717230081558,
      "entropy": 0.46533203125,
      "epoch": 0.136,
      "grad_norm": 0.4445283114910126,
      "kl": 0.0053043365478515625,
      "learning_rate": 2.1911094637307714e-07,
      "loss": -0.0188,
      "reward": 0.7812500298023224,
      "reward_std": 0.3462969809770584,
      "rewards/accuracy_reward": 0.2083333358168602,
      "rewards/format_reward": 0.5729166865348816,
      "step": 119,
      "w_high_ratio": 0.14202763978391886,
      "w_low_ratio": 0.04364374093711376,
      "w_max": 2.151145786046982,
      "w_mean": 1.4133342802524567,
      "w_min": 0.0,
      "w_std": 0.27028138749301434
    },
    {
      "completion_length": 2731.8021240234375,
      "cov_mean": -1.2950695236213505e-06,
      "cov_std": 0.40660279989242554,
      "entropy": 0.45849609375,
      "epoch": 0.13714285714285715,
      "grad_norm": 0.44800451397895813,
      "kl": 0.0019931793212890625,
      "learning_rate": 2.1210398515832536e-07,
      "loss": 0.0072,
      "reward": 0.6770833432674408,
      "reward_std": 0.37227439880371094,
      "rewards/accuracy_reward": 0.17708333395421505,
      "rewards/format_reward": 0.5000000074505806,
      "step": 120,
      "w_high_ratio": 0.06295246630907059,
      "w_low_ratio": 0.051405247300863266,
      "w_max": 1.8777723908424377,
      "w_mean": 1.2852334678173065,
      "w_min": 0.0,
      "w_std": 0.24544718861579895
    },
    {
      "completion_length": 1963.3125610351562,
      "cov_mean": -5.540308461604582e-05,
      "cov_std": 0.36702772229909897,
      "entropy": 0.38525390625,
      "epoch": 0.1382857142857143,
      "grad_norm": 0.5723959803581238,
      "kl": 0.00885772705078125,
      "learning_rate": 2.0528000059645995e-07,
      "loss": -0.076,
      "reward": 1.0104167014360428,
      "reward_std": 0.406433891505003,
      "rewards/accuracy_reward": 0.2291666679084301,
      "rewards/format_reward": 0.7812500149011612,
      "step": 121,
      "w_high_ratio": 0.25104042887687683,
      "w_low_ratio": 0.04047479620203376,
      "w_max": 2.346793830394745,
      "w_mean": 1.5671572387218475,
      "w_min": 0.0,
      "w_std": 0.25984594970941544
    },
    {
      "completion_length": 3143.0208740234375,
      "cov_mean": 1.6946690038821544e-05,
      "cov_std": 0.2671542540192604,
      "entropy": 0.453125,
      "epoch": 0.13942857142857143,
      "grad_norm": 0.1998731791973114,
      "kl": 0.0013976097106933594,
      "learning_rate": 1.986426879955034e-07,
      "loss": -0.0537,
      "reward": 0.7812500149011612,
      "reward_std": 0.4281647428870201,
      "rewards/accuracy_reward": 0.3020833469927311,
      "rewards/format_reward": 0.479166679084301,
      "step": 122,
      "w_high_ratio": 0.01932075247168541,
      "w_low_ratio": 0.030062017496675253,
      "w_max": 1.5524874925613403,
      "w_mean": 1.134686678647995,
      "w_min": 0.0,
      "w_std": 0.1492646411061287
    },
    {
      "completion_length": 3167.3959350585938,
      "cov_mean": -5.797519952466246e-05,
      "cov_std": 0.44271689653396606,
      "entropy": 0.4228515625,
      "epoch": 0.14057142857142857,
      "grad_norm": 0.32559123635292053,
      "kl": 0.001354217529296875,
      "learning_rate": 1.9219564157731844e-07,
      "loss": -0.0282,
      "reward": 0.6562500260770321,
      "reward_std": 0.49003005772829056,
      "rewards/accuracy_reward": 0.1875,
      "rewards/format_reward": 0.4687500111758709,
      "step": 123,
      "w_high_ratio": 0.03850603708997369,
      "w_low_ratio": 0.054318103939294815,
      "w_max": 1.9482422471046448,
      "w_mean": 1.2491904497146606,
      "w_min": 0.0,
      "w_std": 0.273440919816494
    },
    {
      "completion_length": 2679.291748046875,
      "cov_mean": 4.984476254321635e-05,
      "cov_std": 0.3510932922363281,
      "entropy": 0.32421875,
      "epoch": 0.1417142857142857,
      "grad_norm": 0.5072447061538696,
      "kl": 0.0061626434326171875,
      "learning_rate": 1.8594235253127372e-07,
      "loss": 0.0459,
      "reward": 0.8645833730697632,
      "reward_std": 0.44617248326539993,
      "rewards/accuracy_reward": 0.29166667722165585,
      "rewards/format_reward": 0.5729166865348816,
      "step": 124,
      "w_high_ratio": 0.0684627341106534,
      "w_low_ratio": 0.03564309095963836,
      "w_max": 2.0593042075634003,
      "w_mean": 1.2975650131702423,
      "w_min": 0.0,
      "w_std": 0.22546635568141937
    },
    {
      "completion_length": 2886.1666870117188,
      "cov_mean": -3.579185113267158e-05,
      "cov_std": 0.20644951611757278,
      "entropy": 0.365478515625,
      "epoch": 0.14285714285714285,
      "grad_norm": 1.0346649885177612,
      "kl": 0.001129150390625,
      "learning_rate": 1.7988620712370195e-07,
      "loss": -0.0434,
      "reward": 0.6041666772216558,
      "reward_std": 0.16948115080595016,
      "rewards/accuracy_reward": 0.23958333861082792,
      "rewards/format_reward": 0.3645833386108279,
      "step": 125,
      "w_high_ratio": 0.10017836093902588,
      "w_low_ratio": 0.013640805147588253,
      "w_max": 1.8374318480491638,
      "w_mean": 1.238489419221878,
      "w_min": 0.25,
      "w_std": 0.1324586421251297
    },
    {
      "completion_length": 3072.1355590820312,
      "cov_mean": -7.128902507247403e-05,
      "cov_std": 0.268012635409832,
      "entropy": 0.42138671875,
      "epoch": 0.144,
      "grad_norm": 0.24976088106632233,
      "kl": 0.001007080078125,
      "learning_rate": 1.7403048486417868e-07,
      "loss": -0.003,
      "reward": 0.7604166939854622,
      "reward_std": 0.3946729302406311,
      "rewards/accuracy_reward": 0.2395833395421505,
      "rewards/format_reward": 0.5208333432674408,
      "step": 126,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.02901885099709034,
      "w_max": 1.5019842684268951,
      "w_mean": 1.154728651046753,
      "w_min": 0.25,
      "w_std": 0.13917932659387589
    },
    {
      "completion_length": 3639.2084350585938,
      "cov_mean": -4.922010657537612e-05,
      "cov_std": 0.40233808010816574,
      "entropy": 0.4443359375,
      "epoch": 0.14514285714285713,
      "grad_norm": 0.2971266508102417,
      "kl": 0.001190185546875,
      "learning_rate": 1.6837835672960831e-07,
      "loss": 0.0023,
      "reward": 0.291666679084301,
      "reward_std": 0.3814757987856865,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.2604166716337204,
      "step": 127,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.048589578829705715,
      "w_max": 1.546887069940567,
      "w_mean": 1.0961028933525085,
      "w_min": 0.0,
      "w_std": 0.21367743983864784
    },
    {
      "completion_length": 3072.729248046875,
      "cov_mean": 2.080060630760272e-05,
      "cov_std": 0.1757410392165184,
      "entropy": 0.4794921875,
      "epoch": 0.1462857142857143,
      "grad_norm": 0.20411358773708344,
      "kl": 0.0022602081298828125,
      "learning_rate": 1.6293288344708566e-07,
      "loss": -0.0426,
      "reward": 0.8854166865348816,
      "reward_std": 0.23283471912145615,
      "rewards/accuracy_reward": 0.416666679084301,
      "rewards/format_reward": 0.46875,
      "step": 128,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.023187558632344007,
      "w_max": 1.2965095043182373,
      "w_mean": 1.0975826382637024,
      "w_min": 0.5,
      "w_std": 0.09349002316594124
    },
    {
      "completion_length": 3820.5000610351562,
      "cov_mean": -7.538520003436133e-05,
      "cov_std": 0.28777727484703064,
      "entropy": 0.4658203125,
      "epoch": 0.14742857142857144,
      "grad_norm": 0.15165992081165314,
      "kl": 0.0022630691528320312,
      "learning_rate": 1.5769701383645698e-07,
      "loss": 0.0248,
      "reward": 0.3125,
      "reward_std": 0.40841156244277954,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.1875,
      "step": 129,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03958333469927311,
      "w_max": 1.3145957589149475,
      "w_mean": 1.0334198474884033,
      "w_min": 0.25,
      "w_std": 0.15427661687135696
    },
    {
      "completion_length": 3234.9375610351562,
      "cov_mean": -4.075149445270654e-05,
      "cov_std": 0.40202101692557335,
      "entropy": 0.4580078125,
      "epoch": 0.14857142857142858,
      "grad_norm": 0.5406985282897949,
      "kl": 0.0025501251220703125,
      "learning_rate": 1.5267358321348285e-07,
      "loss": 0.0054,
      "reward": 0.43750001676380634,
      "reward_std": 0.48847879469394684,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.30208334140479565,
      "step": 130,
      "w_high_ratio": 0.1068541444838047,
      "w_low_ratio": 0.04362851567566395,
      "w_max": 2.2782379388809204,
      "w_mean": 1.2775286734104156,
      "w_min": 0.0,
      "w_std": 0.27442070841789246
    },
    {
      "completion_length": 2971.3751220703125,
      "cov_mean": 7.5797214776685e-05,
      "cov_std": 0.296669140458107,
      "entropy": 0.42236328125,
      "epoch": 0.14971428571428572,
      "grad_norm": 0.3472649157047272,
      "kl": 0.0027103424072265625,
      "learning_rate": 1.4786531185446452e-07,
      "loss": -0.0555,
      "reward": 0.7812500223517418,
      "reward_std": 0.35348715633153915,
      "rewards/accuracy_reward": 0.3437500074505806,
      "rewards/format_reward": 0.4375000074505806,
      "step": 131,
      "w_high_ratio": 0.03943436220288277,
      "w_low_ratio": 0.02768976055085659,
      "w_max": 1.649653136730194,
      "w_mean": 1.195732295513153,
      "w_min": 0.25,
      "w_std": 0.17519061639904976
    },
    {
      "completion_length": 3084.197998046875,
      "cov_mean": 8.538130737179017e-05,
      "cov_std": 0.4518180638551712,
      "entropy": 0.4111328125,
      "epoch": 0.15085714285714286,
      "grad_norm": 0.3087000846862793,
      "kl": 0.0020132064819335938,
      "learning_rate": 1.432748035231658e-07,
      "loss": 0.0503,
      "reward": 0.6458333507180214,
      "reward_std": 0.5204262509942055,
      "rewards/accuracy_reward": 0.2500000074505806,
      "rewards/format_reward": 0.3958333432674408,
      "step": 132,
      "w_high_ratio": 0.16728225722908974,
      "w_low_ratio": 0.057774459943175316,
      "w_max": 2.160991072654724,
      "w_mean": 1.3055765330791473,
      "w_min": 0.0,
      "w_std": 0.2905358038842678
    },
    {
      "completion_length": 3655.4166870117188,
      "cov_mean": 7.652423300896771e-05,
      "cov_std": 0.2661324590444565,
      "entropy": 0.49365234375,
      "epoch": 0.152,
      "grad_norm": 0.18705269694328308,
      "kl": 0.001617431640625,
      "learning_rate": 1.3890454406082956e-07,
      "loss": 0.075,
      "reward": 0.3645833358168602,
      "reward_std": 0.23336705565452576,
      "rewards/accuracy_reward": 0.1666666679084301,
      "rewards/format_reward": 0.1979166679084301,
      "step": 133,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.04366964288055897,
      "w_max": 1.3597778677940369,
      "w_mean": 1.095180168747902,
      "w_min": 0.5,
      "w_std": 0.1226506233215332
    },
    {
      "completion_length": 2935.791748046875,
      "cov_mean": 4.5339866119320504e-05,
      "cov_std": 0.3691224604845047,
      "entropy": 0.47900390625,
      "epoch": 0.15314285714285714,
      "grad_norm": 0.3127249777317047,
      "kl": 0.0026226043701171875,
      "learning_rate": 1.3475690004005097e-07,
      "loss": 0.0052,
      "reward": 0.739583358168602,
      "reward_std": 0.5291576012969017,
      "rewards/accuracy_reward": 0.25000000931322575,
      "rewards/format_reward": 0.4895833507180214,
      "step": 134,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.049329387256875634,
      "w_max": 1.5319222509860992,
      "w_mean": 1.2102845907211304,
      "w_min": 0.0,
      "w_std": 0.20571278221905231
    },
    {
      "completion_length": 2150.0625610351562,
      "cov_mean": -6.792098974983674e-05,
      "cov_std": 0.41625837981700897,
      "entropy": 0.40966796875,
      "epoch": 0.15428571428571428,
      "grad_norm": 0.5556284189224243,
      "kl": 0.00433349609375,
      "learning_rate": 1.308341174832359e-07,
      "loss": -0.0542,
      "reward": 1.2187500149011612,
      "reward_std": 0.4045410081744194,
      "rewards/accuracy_reward": 0.5312500074505806,
      "rewards/format_reward": 0.6875000074505806,
      "step": 135,
      "w_high_ratio": 0.1767500415444374,
      "w_low_ratio": 0.04722660221159458,
      "w_max": 2.584423005580902,
      "w_mean": 1.3822406232357025,
      "w_min": 3.783363098268088e-33,
      "w_std": 0.2943294197320938
    },
    {
      "completion_length": 3219.2188110351562,
      "cov_mean": 7.927787930839258e-05,
      "cov_std": 0.36336907744407654,
      "entropy": 0.36572265625,
      "epoch": 0.15542857142857142,
      "grad_norm": 0.4164927303791046,
      "kl": 0.0019321441650390625,
      "learning_rate": 1.2713832064634125e-07,
      "loss": -0.1128,
      "reward": 0.9062500298023224,
      "reward_std": 0.520418331027031,
      "rewards/accuracy_reward": 0.4375000074505806,
      "rewards/format_reward": 0.4687500149011612,
      "step": 136,
      "w_high_ratio": 0.09939645975828171,
      "w_low_ratio": 0.0331453662365675,
      "w_max": 1.8468493521213531,
      "w_mean": 1.2761128842830658,
      "w_min": 0.25,
      "w_std": 0.22711537778377533
    },
    {
      "completion_length": 3419.6458740234375,
      "cov_mean": -3.98908814531751e-05,
      "cov_std": 0.4879545792937279,
      "entropy": 0.36328125,
      "epoch": 0.15657142857142858,
      "grad_norm": 0.6263577938079834,
      "kl": 0.0011224746704101562,
      "learning_rate": 1.2367151086855187e-07,
      "loss": 0.0354,
      "reward": 0.4479166716337204,
      "reward_std": 0.5504394620656967,
      "rewards/accuracy_reward": 0.10416667070239782,
      "rewards/format_reward": 0.3437500074505806,
      "step": 137,
      "w_high_ratio": 0.040824100375175476,
      "w_low_ratio": 0.054094865918159485,
      "w_max": 1.8979838490486145,
      "w_mean": 1.1621877253055573,
      "w_min": 0.0,
      "w_std": 0.2735421061515808
    },
    {
      "completion_length": 3128.0626220703125,
      "cov_mean": -2.278413830936188e-05,
      "cov_std": 0.27040576189756393,
      "entropy": 0.38330078125,
      "epoch": 0.15771428571428572,
      "grad_norm": 0.9205570816993713,
      "kl": 0.049633026123046875,
      "learning_rate": 1.2043556548852063e-07,
      "loss": 0.036,
      "reward": 0.5520833358168602,
      "reward_std": 0.3116639107465744,
      "rewards/accuracy_reward": 0.1666666716337204,
      "rewards/format_reward": 0.385416679084301,
      "step": 138,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.038355547934770584,
      "w_max": 1.4525592625141144,
      "w_mean": 1.1665604412555695,
      "w_min": 0.25,
      "w_std": 0.14510583132505417
    },
    {
      "completion_length": 3528.5313110351562,
      "cov_mean": 3.168399962305557e-05,
      "cov_std": 0.5140600129961967,
      "entropy": 0.498046875,
      "epoch": 0.15885714285714286,
      "grad_norm": 0.4198448061943054,
      "kl": 0.0019092559814453125,
      "learning_rate": 1.1743223682775649e-07,
      "loss": -0.0322,
      "reward": 0.5104166865348816,
      "reward_std": 0.562984399497509,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.3750000186264515,
      "step": 139,
      "w_high_ratio": 0.04249488562345505,
      "w_low_ratio": 0.06252239271998405,
      "w_max": 1.7476003468036652,
      "w_mean": 1.1608193814754486,
      "w_min": 0.0,
      "w_std": 0.2752341143786907
    },
    {
      "completion_length": 3422.666748046875,
      "cov_mean": -5.857029464095831e-05,
      "cov_std": 0.4820387288928032,
      "entropy": 0.51123046875,
      "epoch": 0.16,
      "grad_norm": 0.747706413269043,
      "kl": 0.00482940673828125,
      "learning_rate": 1.1466315124171128e-07,
      "loss": -0.0162,
      "reward": 0.4791666865348816,
      "reward_std": 0.521266907453537,
      "rewards/accuracy_reward": 0.18750000186264515,
      "rewards/format_reward": 0.2916666753590107,
      "step": 140,
      "w_high_ratio": 0.09309478104114532,
      "w_low_ratio": 0.05631279572844505,
      "w_max": 2.1824983656406403,
      "w_mean": 1.2857783138751984,
      "w_min": 4.5542200090556555e-45,
      "w_std": 0.3347093164920807
    },
    {
      "completion_length": 3233.166748046875,
      "cov_mean": 2.940024387498852e-06,
      "cov_std": 0.45166684687137604,
      "entropy": 0.40185546875,
      "epoch": 0.16114285714285714,
      "grad_norm": 0.3896685838699341,
      "kl": 0.0029087066650390625,
      "learning_rate": 1.1212980823907929e-07,
      "loss": -0.0202,
      "reward": 0.6875000149011612,
      "reward_std": 0.5667420700192451,
      "rewards/accuracy_reward": 0.21875000651925802,
      "rewards/format_reward": 0.4687500149011612,
      "step": 141,
      "w_high_ratio": 0.04078603908419609,
      "w_low_ratio": 0.0461601298302412,
      "w_max": 1.661450743675232,
      "w_mean": 1.181685209274292,
      "w_min": 0.0,
      "w_std": 0.24826455861330032
    },
    {
      "completion_length": 3014.7084045410156,
      "cov_mean": 1.010456662697834e-05,
      "cov_std": 0.35788750648498535,
      "entropy": 0.43212890625,
      "epoch": 0.16228571428571428,
      "grad_norm": 0.37326905131340027,
      "kl": 0.0032243728637695312,
      "learning_rate": 1.0983357966978745e-07,
      "loss": 0.0113,
      "reward": 0.8020833730697632,
      "reward_std": 0.4403490200638771,
      "rewards/accuracy_reward": 0.23958334233611822,
      "rewards/format_reward": 0.5625000074505806,
      "step": 142,
      "w_high_ratio": 0.07575653120875359,
      "w_low_ratio": 0.04324484569951892,
      "w_max": 1.8279287815093994,
      "w_mean": 1.3297627568244934,
      "w_min": 0.0,
      "w_std": 0.21703040227293968
    },
    {
      "completion_length": 3143.479248046875,
      "cov_mean": 4.2679124817368574e-05,
      "cov_std": 0.5287895128130913,
      "entropy": 0.4609375,
      "epoch": 0.16342857142857142,
      "grad_norm": 0.8503015637397766,
      "kl": 0.004160881042480469,
      "learning_rate": 1.0777570898211405e-07,
      "loss": -0.0551,
      "reward": 0.5208333507180214,
      "reward_std": 0.4103339910507202,
      "rewards/accuracy_reward": 0.1041666716337204,
      "rewards/format_reward": 0.416666679084301,
      "step": 143,
      "w_high_ratio": 0.06985687837004662,
      "w_low_ratio": 0.06702116876840591,
      "w_max": 2.2525435388088226,
      "w_mean": 1.2808727622032166,
      "w_min": 0.0,
      "w_std": 0.33132829889655113
    },
    {
      "completion_length": 3187.3750610351562,
      "cov_mean": -3.3114460165961646e-05,
      "cov_std": 0.4341953620314598,
      "entropy": 0.40185546875,
      "epoch": 0.16457142857142856,
      "grad_norm": 0.36935943365097046,
      "kl": 0.0021028518676757812,
      "learning_rate": 1.0595731054933934e-07,
      "loss": 0.0152,
      "reward": 0.6250000223517418,
      "reward_std": 0.54064517095685,
      "rewards/accuracy_reward": 0.27083333395421505,
      "rewards/format_reward": 0.3541666716337204,
      "step": 144,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.04692948702722788,
      "w_max": 1.7171835899353027,
      "w_mean": 1.1371115744113922,
      "w_min": 0.0,
      "w_std": 0.24135740101337433
    },
    {
      "completion_length": 2363.416748046875,
      "cov_mean": 7.833678682800382e-05,
      "cov_std": 0.23034526035189629,
      "entropy": 0.3486328125,
      "epoch": 0.1657142857142857,
      "grad_norm": 0.32663026452064514,
      "kl": 0.0025043487548828125,
      "learning_rate": 1.0437936906629334e-07,
      "loss": 0.0397,
      "reward": 0.916666692122817,
      "reward_std": 0.28357625752687454,
      "rewards/accuracy_reward": 0.3125000111758709,
      "rewards/format_reward": 0.6041666772216558,
      "step": 145,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.027998102828860283,
      "w_max": 1.3861334919929504,
      "w_mean": 1.1757619678974152,
      "w_min": 0.2742290794849396,
      "w_std": 0.13859782367944717
    },
    {
      "completion_length": 2963.791748046875,
      "cov_mean": 3.222269015168422e-05,
      "cov_std": 0.2941744774580002,
      "entropy": 0.43359375,
      "epoch": 0.16685714285714287,
      "grad_norm": 0.466317743062973,
      "kl": 0.0017948150634765625,
      "learning_rate": 1.0304273901612565e-07,
      "loss": -0.0398,
      "reward": 0.5312500074505806,
      "reward_std": 0.31367800384759903,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.4687500074505806,
      "step": 146,
      "w_high_ratio": 0.08747749403119087,
      "w_low_ratio": 0.03374567721039057,
      "w_max": 2.10350301861763,
      "w_mean": 1.2755134403705597,
      "w_min": 0.25,
      "w_std": 0.2049087956547737
    },
    {
      "completion_length": 3316.1875610351562,
      "cov_mean": -2.5228303002222674e-07,
      "cov_std": 0.2783627863973379,
      "entropy": 0.42431640625,
      "epoch": 0.168,
      "grad_norm": 0.2964071035385132,
      "kl": 0.00269317626953125,
      "learning_rate": 1.0194814420758804e-07,
      "loss": -0.0063,
      "reward": 0.4687500102445483,
      "reward_std": 0.41155891865491867,
      "rewards/accuracy_reward": 0.1562500037252903,
      "rewards/format_reward": 0.3125000102445483,
      "step": 147,
      "w_high_ratio": 0.08768598735332489,
      "w_low_ratio": 0.027966859750449657,
      "w_max": 1.8361600935459137,
      "w_mean": 1.2343448996543884,
      "w_min": 0.0,
      "w_std": 0.19136979151517153
    },
    {
      "completion_length": 2667.0625,
      "cov_mean": 6.741791366948746e-05,
      "cov_std": 0.2421601451933384,
      "entropy": 0.3486328125,
      "epoch": 0.16914285714285715,
      "grad_norm": 0.5170913338661194,
      "kl": 0.0018463134765625,
      "learning_rate": 1.0109617738307911e-07,
      "loss": -0.0546,
      "reward": 0.708333358168602,
      "reward_std": 0.22040386497974396,
      "rewards/accuracy_reward": 0.1666666679084301,
      "rewards/format_reward": 0.5416666865348816,
      "step": 148,
      "w_high_ratio": 0.10121732205152512,
      "w_low_ratio": 0.022710513323545456,
      "w_max": 2.0310742557048798,
      "w_mean": 1.4218811392784119,
      "w_min": 0.25,
      "w_std": 0.20712272450327873
    },
    {
      "completion_length": 3187.947998046875,
      "cov_mean": -3.065193595830351e-05,
      "cov_std": 0.278855100274086,
      "entropy": 0.34326171875,
      "epoch": 0.1702857142857143,
      "grad_norm": 0.3262748122215271,
      "kl": 0.011348247528076172,
      "learning_rate": 1.0048729989766394e-07,
      "loss": -0.0407,
      "reward": 0.656250013038516,
      "reward_std": 0.2951196879148483,
      "rewards/accuracy_reward": 0.1979166716337204,
      "rewards/format_reward": 0.45833334140479565,
      "step": 149,
      "w_high_ratio": 0.006521851755678654,
      "w_low_ratio": 0.021616162732243538,
      "w_max": 1.6156696677207947,
      "w_mean": 1.1841600239276886,
      "w_min": 0.0,
      "w_std": 0.13553307205438614
    },
    {
      "completion_length": 3107.3125,
      "cov_mean": 1.709405751171289e-05,
      "cov_std": 0.34910060465335846,
      "entropy": 0.3984375,
      "epoch": 0.17142857142857143,
      "grad_norm": 0.33226433396339417,
      "kl": 0.0028133392333984375,
      "learning_rate": 1.0012184146924223e-07,
      "loss": 0.0548,
      "reward": 0.520833358168602,
      "reward_std": 0.40651097148656845,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.3854166679084301,
      "step": 150,
      "w_high_ratio": 0.05817456915974617,
      "w_low_ratio": 0.04275808576494455,
      "w_max": 1.5990401804447174,
      "w_mean": 1.167884886264801,
      "w_min": 0.25,
      "w_std": 0.18246712163090706
    },
    {
      "epoch": 0.17142857142857143,
      "step": 150,
      "total_flos": 0.0,
      "train_loss": -0.007013439348277946,
      "train_runtime": 12590.1794,
      "train_samples_per_second": 1.144,
      "train_steps_per_second": 0.012
    }
  ],
  "logging_steps": 1,
  "max_steps": 150,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 6,
  "trial_name": null,
  "trial_params": null
}