{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.11428571428571428,
  "eval_steps": 500,
  "global_step": 100,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 2700.4271850585938,
      "cov_mean": -2.6832926778297406e-05,
      "cov_std": 0.24635104648768902,
      "entropy": 0.36865234375,
      "epoch": 0.001142857142857143,
      "grad_norm": 0.35615867376327515,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": 0.0696,
      "reward": 0.7604166893288493,
      "reward_std": 0.4268697127699852,
      "rewards/accuracy_reward": 0.25000001303851604,
      "rewards/format_reward": 0.5104166669771075,
      "step": 1,
      "w_high_ratio": 0.2208261415362358,
      "w_low_ratio": 0.027151118498295546,
      "w_max": 2.1915207505226135,
      "w_mean": 1.4711343348026276,
      "w_min": 1.404075949984986e-37,
      "w_std": 0.24041971936821938
    },
    {
      "completion_length": 3127.3958435058594,
      "cov_mean": -1.8215427189716138e-05,
      "cov_std": 0.18336841650307178,
      "entropy": 0.353515625,
      "epoch": 0.002285714285714286,
      "grad_norm": 0.18010225892066956,
      "kl": 0.0,
      "learning_rate": 1e-07,
      "loss": 0.0533,
      "reward": 0.6458333637565374,
      "reward_std": 0.4249730706214905,
      "rewards/accuracy_reward": 0.2812500102445483,
      "rewards/format_reward": 0.3645833386108279,
      "step": 2,
      "w_high_ratio": 0.05701034888625145,
      "w_low_ratio": 0.023528859252110124,
      "w_max": 1.811183512210846,
      "w_mean": 1.2113382518291473,
      "w_min": 0.0,
      "w_std": 0.15613791532814503
    },
    {
      "completion_length": 3691.0626220703125,
      "cov_mean": 2.796226033296989e-05,
      "cov_std": 0.1637928392738104,
      "entropy": 0.44189453125,
      "epoch": 0.0034285714285714284,
      "grad_norm": 0.1356951743364334,
      "kl": 3.916025161743164e-05,
      "learning_rate": 2e-07,
      "loss": 0.052,
      "reward": 0.19791667256504297,
      "reward_std": 0.3607826754450798,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.14583333674818277,
      "step": 3,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.02235229848884046,
      "w_max": 1.460817277431488,
      "w_mean": 1.082369714975357,
      "w_min": 3.3280838527714405e-44,
      "w_std": 0.12309953197836876
    },
    {
      "completion_length": 2353.2709350585938,
      "cov_mean": 1.0425418167869793e-05,
      "cov_std": 0.3036706894636154,
      "entropy": 0.41259765625,
      "epoch": 0.004571428571428572,
      "grad_norm": 0.190170019865036,
      "kl": 3.3348798751831055e-05,
      "learning_rate": 3e-07,
      "loss": 0.0459,
      "reward": 0.8750000149011612,
      "reward_std": 0.5107106417417526,
      "rewards/accuracy_reward": 0.1875000037252903,
      "rewards/format_reward": 0.6875000298023224,
      "step": 4,
      "w_high_ratio": 0.2652290016412735,
      "w_low_ratio": 0.034206886775791645,
      "w_max": 2.106997400522232,
      "w_mean": 1.5420070886611938,
      "w_min": 2.4617042843759845e-36,
      "w_std": 0.2812090367078781
    },
    {
      "completion_length": 3485.1771850585938,
      "cov_mean": 3.2382055223934003e-06,
      "cov_std": 0.29665667191147804,
      "entropy": 0.4609375,
      "epoch": 0.005714285714285714,
      "grad_norm": 0.2197088897228241,
      "kl": 4.2125582695007324e-05,
      "learning_rate": 4e-07,
      "loss": 0.0803,
      "reward": 0.46875001303851604,
      "reward_std": 0.5515270829200745,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.35416667722165585,
      "step": 5,
      "w_high_ratio": 0.008333034813404083,
      "w_low_ratio": 0.04545952333137393,
      "w_max": 1.5202394425868988,
      "w_mean": 1.1503158807754517,
      "w_min": 5.693325166185118e-29,
      "w_std": 0.23378031328320503
    },
    {
      "completion_length": 3451.2500610351562,
      "cov_mean": -4.464495305001037e-05,
      "cov_std": 0.236886378377676,
      "entropy": 0.46142578125,
      "epoch": 0.006857142857142857,
      "grad_norm": 0.13218647241592407,
      "kl": 4.482269287109375e-05,
      "learning_rate": 5e-07,
      "loss": 0.0517,
      "reward": 0.3645833507180214,
      "reward_std": 0.515114888548851,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.28125000186264515,
      "step": 6,
      "w_high_ratio": 0.0625,
      "w_low_ratio": 0.031897591426968575,
      "w_max": 1.5891262888908386,
      "w_mean": 1.1359511613845825,
      "w_min": 0.0,
      "w_std": 0.14766533859074116
    },
    {
      "completion_length": 3224.3125610351562,
      "cov_mean": 4.349886694399174e-06,
      "cov_std": 0.3991788253188133,
      "entropy": 0.38671875,
      "epoch": 0.008,
      "grad_norm": 0.22412240505218506,
      "kl": 2.1651387214660645e-05,
      "learning_rate": 6e-07,
      "loss": 0.0742,
      "reward": 0.8541666865348816,
      "reward_std": 0.6870906725525856,
      "rewards/accuracy_reward": 0.2291666716337204,
      "rewards/format_reward": 0.6250000149011612,
      "step": 7,
      "w_high_ratio": 0.047733694314956665,
      "w_low_ratio": 0.053672163281589746,
      "w_max": 1.5988431572914124,
      "w_mean": 1.2651265263557434,
      "w_min": 6.1929912552473734e-37,
      "w_std": 0.2889493927359581
    },
    {
      "completion_length": 2800.9583740234375,
      "cov_mean": 1.0622998161124997e-06,
      "cov_std": 0.15430260822176933,
      "entropy": 0.33740234375,
      "epoch": 0.009142857142857144,
      "grad_norm": 0.11328813433647156,
      "kl": 1.7002224922180176e-05,
      "learning_rate": 7e-07,
      "loss": 0.0184,
      "reward": 0.8958333730697632,
      "reward_std": 0.25296592339873314,
      "rewards/accuracy_reward": 0.4062500149011612,
      "rewards/format_reward": 0.4895833432674408,
      "step": 8,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.01592865912243724,
      "w_max": 1.5333127677440643,
      "w_mean": 1.2431240677833557,
      "w_min": 0.25,
      "w_std": 0.11287659406661987
    },
    {
      "completion_length": 3369.791748046875,
      "cov_mean": -2.2509159407491097e-05,
      "cov_std": 0.20683829113841057,
      "entropy": 0.45263671875,
      "epoch": 0.010285714285714285,
      "grad_norm": 0.1632954180240631,
      "kl": 4.3064355850219727e-05,
      "learning_rate": 8e-07,
      "loss": 0.0417,
      "reward": 0.4583333507180214,
      "reward_std": 0.3903508894145489,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.3437500074505806,
      "step": 9,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.029203591868281364,
      "w_max": 1.6281995177268982,
      "w_mean": 1.1540252268314362,
      "w_min": 1.9273542721946577e-23,
      "w_std": 0.15844954177737236
    },
    {
      "completion_length": 2794.2291870117188,
      "cov_mean": 7.258828873091261e-06,
      "cov_std": 0.22622444108128548,
      "entropy": 0.34716796875,
      "epoch": 0.011428571428571429,
      "grad_norm": 0.11491074413061142,
      "kl": 2.664327621459961e-05,
      "learning_rate": 9e-07,
      "loss": 0.0765,
      "reward": 0.6145833432674408,
      "reward_std": 0.4795500487089157,
      "rewards/accuracy_reward": 0.16666667070239782,
      "rewards/format_reward": 0.4479166828095913,
      "step": 10,
      "w_high_ratio": 0.1683393381536007,
      "w_low_ratio": 0.03143396740779281,
      "w_max": 1.8651617467403412,
      "w_mean": 1.2822044789791107,
      "w_min": 2.2624703592113335e-38,
      "w_std": 0.20682579837739468
    },
    {
      "completion_length": 3703.197998046875,
      "cov_mean": -2.311449361513951e-05,
      "cov_std": 0.18362887762486935,
      "entropy": 0.39697265625,
      "epoch": 0.012571428571428572,
      "grad_norm": 0.12180526554584503,
      "kl": 2.73287296295166e-05,
      "learning_rate": 1e-06,
      "loss": 0.0401,
      "reward": 0.26041667722165585,
      "reward_std": 0.3744332268834114,
      "rewards/accuracy_reward": 0.10416667256504297,
      "rewards/format_reward": 0.15625000558793545,
      "step": 11,
      "w_high_ratio": 0.027614232152700424,
      "w_low_ratio": 0.024976021610200405,
      "w_max": 1.3482708036899567,
      "w_mean": 1.0835402309894562,
      "w_min": 0.25,
      "w_std": 0.12668619584292173
    },
    {
      "completion_length": 2611.260498046875,
      "cov_mean": 1.0368909215685562e-05,
      "cov_std": 0.22750693373382092,
      "entropy": 0.3984375,
      "epoch": 0.013714285714285714,
      "grad_norm": 0.1790972799062729,
      "kl": 2.802908420562744e-05,
      "learning_rate": 9.997258721585931e-07,
      "loss": 0.0576,
      "reward": 0.7395833805203438,
      "reward_std": 0.4462515264749527,
      "rewards/accuracy_reward": 0.1354166716337204,
      "rewards/format_reward": 0.604166679084301,
      "step": 12,
      "w_high_ratio": 0.10206323117017746,
      "w_low_ratio": 0.0320228124037385,
      "w_max": 2.1756480634212494,
      "w_mean": 1.4741427898406982,
      "w_min": 4.420129648185005e-23,
      "w_std": 0.23598888516426086
    },
    {
      "completion_length": 3224.041748046875,
      "cov_mean": -2.2661331968265586e-05,
      "cov_std": 0.15935274586081505,
      "entropy": 0.38427734375,
      "epoch": 0.014857142857142857,
      "grad_norm": 0.24569930136203766,
      "kl": 1.7702579498291016e-05,
      "learning_rate": 9.989038226169207e-07,
      "loss": 0.0498,
      "reward": 0.604166679084301,
      "reward_std": 0.30622391402721405,
      "rewards/accuracy_reward": 0.21875000558793545,
      "rewards/format_reward": 0.385416679084301,
      "step": 13,
      "w_high_ratio": 0.20555464923381805,
      "w_low_ratio": 0.01726952870376408,
      "w_max": 1.8901410400867462,
      "w_mean": 1.31855970621109,
      "w_min": 1.1411503146395655e-35,
      "w_std": 0.14450976066291332
    },
    {
      "completion_length": 3125.197998046875,
      "cov_mean": -2.9986793833813863e-05,
      "cov_std": 0.18112273141741753,
      "entropy": 0.3623046875,
      "epoch": 0.016,
      "grad_norm": 0.14652569591999054,
      "kl": 7.249414920806885e-06,
      "learning_rate": 9.975348529157229e-07,
      "loss": 0.0596,
      "reward": 0.5000000111758709,
      "reward_std": 0.3975026085972786,
      "rewards/accuracy_reward": 0.13541667070239782,
      "rewards/format_reward": 0.3645833432674408,
      "step": 14,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.026353970635682344,
      "w_max": 1.6898008584976196,
      "w_mean": 1.1696374714374542,
      "w_min": 2.786338774190119e-34,
      "w_std": 0.17131789773702621
    },
    {
      "completion_length": 2945.3959350585938,
      "cov_mean": 3.4355500702076824e-06,
      "cov_std": 0.18097041826695204,
      "entropy": 0.37109375,
      "epoch": 0.017142857142857144,
      "grad_norm": 0.08377102017402649,
      "kl": 2.804398536682129e-05,
      "learning_rate": 9.956206309337066e-07,
      "loss": 0.0282,
      "reward": 0.6875000149011612,
      "reward_std": 0.37770550325512886,
      "rewards/accuracy_reward": 0.2395833395421505,
      "rewards/format_reward": 0.4479166716337204,
      "step": 15,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.027025693794712424,
      "w_max": 1.4225987792015076,
      "w_mean": 1.1180275976657867,
      "w_min": 8.951121255272305e-16,
      "w_std": 0.14841708727180958
    },
    {
      "completion_length": 3842.3646240234375,
      "cov_mean": -3.1302830393542536e-05,
      "cov_std": 0.16068686172366142,
      "entropy": 0.458984375,
      "epoch": 0.018285714285714287,
      "grad_norm": 0.1221930980682373,
      "kl": 2.4199485778808594e-05,
      "learning_rate": 9.931634888554935e-07,
      "loss": 0.0431,
      "reward": 0.1562500037252903,
      "reward_std": 0.3155686669051647,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.10416666977107525,
      "step": 16,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.022678226232528687,
      "w_max": 1.1968038976192474,
      "w_mean": 1.0266980826854706,
      "w_min": 0.25,
      "w_std": 0.10604305937886238
    },
    {
      "completion_length": 2433.1875915527344,
      "cov_mean": 2.4951528757810593e-05,
      "cov_std": 0.27749199233949184,
      "entropy": 0.44970703125,
      "epoch": 0.019428571428571427,
      "grad_norm": 0.13208113610744476,
      "kl": 5.91278076171875e-05,
      "learning_rate": 9.901664203302124e-07,
      "loss": -0.0048,
      "reward": 0.8854166865348816,
      "reward_std": 0.4504813477396965,
      "rewards/accuracy_reward": 0.2708333395421505,
      "rewards/format_reward": 0.6145833358168602,
      "step": 17,
      "w_high_ratio": 0.171035997569561,
      "w_low_ratio": 0.03585993289016187,
      "w_max": 2.199991285800934,
      "w_mean": 1.4317797720432281,
      "w_min": 0.25,
      "w_std": 0.24799126759171486
    },
    {
      "completion_length": 3167.4791870117188,
      "cov_mean": -2.742706919889315e-05,
      "cov_std": 0.2498251087963581,
      "entropy": 0.369140625,
      "epoch": 0.02057142857142857,
      "grad_norm": 0.15079385042190552,
      "kl": 2.0952895283699036e-05,
      "learning_rate": 9.866330768241983e-07,
      "loss": 0.0577,
      "reward": 0.5729166939854622,
      "reward_std": 0.5097959190607071,
      "rewards/accuracy_reward": 0.1666666716337204,
      "rewards/format_reward": 0.4062500149011612,
      "step": 18,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.034274401143193245,
      "w_max": 1.457490622997284,
      "w_mean": 1.1403506994247437,
      "w_min": 0.0,
      "w_std": 0.16276290826499462
    },
    {
      "completion_length": 3139.635498046875,
      "cov_mean": 1.529891596874222e-05,
      "cov_std": 0.13174043968319893,
      "entropy": 0.39208984375,
      "epoch": 0.021714285714285714,
      "grad_norm": 0.08939936012029648,
      "kl": 5.739927291870117e-05,
      "learning_rate": 9.825677631722435e-07,
      "loss": 0.0233,
      "reward": 0.8541666828095913,
      "reward_std": 0.3213166669011116,
      "rewards/accuracy_reward": 0.3750000111758709,
      "rewards/format_reward": 0.47916666977107525,
      "step": 19,
      "w_high_ratio": 0.035190850496292114,
      "w_low_ratio": 0.016721592284739017,
      "w_max": 1.647162914276123,
      "w_mean": 1.2506683766841888,
      "w_min": 0.25,
      "w_std": 0.09711403585970402
    },
    {
      "completion_length": 2464.385498046875,
      "cov_mean": 3.157450896651426e-05,
      "cov_std": 0.283736914396286,
      "entropy": 0.3369140625,
      "epoch": 0.022857142857142857,
      "grad_norm": 0.3488742411136627,
      "kl": 9.429454803466797e-05,
      "learning_rate": 9.779754323328192e-07,
      "loss": 0.0914,
      "reward": 0.9375000596046448,
      "reward_std": 0.4943716749548912,
      "rewards/accuracy_reward": 0.2604166716337204,
      "rewards/format_reward": 0.6770833656191826,
      "step": 20,
      "w_high_ratio": 0.0566110759973526,
      "w_low_ratio": 0.031579687260091305,
      "w_max": 2.3184494078159332,
      "w_mean": 1.4481623768806458,
      "w_min": 0.0,
      "w_std": 0.28026906587183475
    },
    {
      "completion_length": 2847.21875,
      "cov_mean": -2.243259518763807e-05,
      "cov_std": 0.18684318475425243,
      "entropy": 0.423828125,
      "epoch": 0.024,
      "grad_norm": 0.13671471178531647,
      "kl": 0.00033351778984069824,
      "learning_rate": 9.728616793536587e-07,
      "loss": 0.0498,
      "reward": 0.6562500204890966,
      "reward_std": 0.38981083035469055,
      "rewards/accuracy_reward": 0.18750000465661287,
      "rewards/format_reward": 0.4687500027939677,
      "step": 21,
      "w_high_ratio": 0.08715118188410997,
      "w_low_ratio": 0.021108672255650163,
      "w_max": 1.9608261287212372,
      "w_mean": 1.387522131204605,
      "w_min": 4.576730842832761e-23,
      "w_std": 0.14338573440909386
    },
    {
      "completion_length": 1849.3542175292969,
      "cov_mean": -5.1019123930018395e-05,
      "cov_std": 0.208794716745615,
      "entropy": 0.3994140625,
      "epoch": 0.025142857142857144,
      "grad_norm": 0.1823095828294754,
      "kl": 0.00039577484130859375,
      "learning_rate": 9.672327345550543e-07,
      "loss": 0.0503,
      "reward": 1.1041666716337204,
      "reward_std": 0.363413717597723,
      "rewards/accuracy_reward": 0.2708333432674408,
      "rewards/format_reward": 0.833333358168602,
      "step": 22,
      "w_high_ratio": 0.3683718554675579,
      "w_low_ratio": 0.028410385129973292,
      "w_max": 2.5447845458984375,
      "w_mean": 1.7170847058296204,
      "w_min": 1.7296456515038733e-32,
      "w_std": 0.18113290891051292
    },
    {
      "completion_length": 2786.604217529297,
      "cov_mean": 6.156731569717522e-05,
      "cov_std": 0.21581846103072166,
      "entropy": 0.3828125,
      "epoch": 0.026285714285714287,
      "grad_norm": 0.10202132165431976,
      "kl": 0.00020551681518554688,
      "learning_rate": 9.610954559391704e-07,
      "loss": 0.0436,
      "reward": 0.708333358168602,
      "reward_std": 0.47307053953409195,
      "rewards/accuracy_reward": 0.22916667722165585,
      "rewards/format_reward": 0.4791666716337204,
      "step": 23,
      "w_high_ratio": 0.05155515298247337,
      "w_low_ratio": 0.03038623696193099,
      "w_max": 1.8245242238044739,
      "w_mean": 1.2686880826950073,
      "w_min": 2.0108632963061125e-43,
      "w_std": 0.19650068879127502
    },
    {
      "completion_length": 2932.14599609375,
      "cov_mean": 3.685860542645969e-05,
      "cov_std": 0.19567562174052,
      "entropy": 0.35986328125,
      "epoch": 0.027428571428571427,
      "grad_norm": 0.11536505818367004,
      "kl": 0.00012372806668281555,
      "learning_rate": 9.54457320834625e-07,
      "loss": 0.0259,
      "reward": 0.8229167014360428,
      "reward_std": 0.41623104363679886,
      "rewards/accuracy_reward": 0.2708333358168602,
      "rewards/format_reward": 0.5520833507180214,
      "step": 24,
      "w_high_ratio": 0.05616182088851929,
      "w_low_ratio": 0.025760386954061687,
      "w_max": 1.7301380336284637,
      "w_mean": 1.2399356663227081,
      "w_min": 0.25,
      "w_std": 0.16614723671227694
    },
    {
      "completion_length": 2980.104248046875,
      "cov_mean": 5.5617931593587855e-06,
      "cov_std": 0.20383853651583195,
      "entropy": 0.43408203125,
      "epoch": 0.02857142857142857,
      "grad_norm": 0.13895268738269806,
      "kl": 0.0003798753023147583,
      "learning_rate": 9.473264167865171e-07,
      "loss": 0.0328,
      "reward": 0.6145833609625697,
      "reward_std": 0.4326799139380455,
      "rewards/accuracy_reward": 0.20833334140479565,
      "rewards/format_reward": 0.4062500102445483,
      "step": 25,
      "w_high_ratio": 0.15471260249614716,
      "w_low_ratio": 0.027910931850783527,
      "w_max": 1.878886878490448,
      "w_mean": 1.3457823991775513,
      "w_min": 2.1938871947043534e-19,
      "w_std": 0.21430648770183325
    },
    {
      "completion_length": 3121.822998046875,
      "cov_mean": -1.5570902263561948e-05,
      "cov_std": 0.12479476444423199,
      "entropy": 0.41845703125,
      "epoch": 0.029714285714285714,
      "grad_norm": 0.055989839136600494,
      "kl": 5.704164505004883e-05,
      "learning_rate": 9.397114317029974e-07,
      "loss": 0.006,
      "reward": 0.7395833656191826,
      "reward_std": 0.2778088226914406,
      "rewards/accuracy_reward": 0.25000000558793545,
      "rewards/format_reward": 0.4895833358168602,
      "step": 26,
      "w_high_ratio": 0.05233287438750267,
      "w_low_ratio": 0.016803464153781533,
      "w_max": 1.667470008134842,
      "w_mean": 1.2069356143474579,
      "w_min": 1.8282741064045888e-40,
      "w_std": 0.12309898342937231
    },
    {
      "completion_length": 3419.697998046875,
      "cov_mean": -8.16069814391085e-06,
      "cov_std": 0.2326441928744316,
      "entropy": 0.45654296875,
      "epoch": 0.030857142857142857,
      "grad_norm": 0.1154065951704979,
      "kl": 0.0001367814838886261,
      "learning_rate": 9.316216432703916e-07,
      "loss": 0.0629,
      "reward": 0.4687500223517418,
      "reward_std": 0.4522514268755913,
      "rewards/accuracy_reward": 0.11458333861082792,
      "rewards/format_reward": 0.3541666716337204,
      "step": 27,
      "w_high_ratio": 0.125,
      "w_low_ratio": 0.03899317281320691,
      "w_max": 1.7131148278713226,
      "w_mean": 1.2755843102931976,
      "w_min": 2.3244726053753363e-31,
      "w_std": 0.15411211177706718
    },
    {
      "completion_length": 3003.3334350585938,
      "cov_mean": 3.4548415897006635e-06,
      "cov_std": 0.18210824206471443,
      "entropy": 0.40576171875,
      "epoch": 0.032,
      "grad_norm": 0.10329318046569824,
      "kl": 0.00033906102180480957,
      "learning_rate": 9.230669076497687e-07,
      "loss": 0.0466,
      "reward": 0.729166679084301,
      "reward_std": 0.4190382733941078,
      "rewards/accuracy_reward": 0.291666679084301,
      "rewards/format_reward": 0.4375000149011612,
      "step": 28,
      "w_high_ratio": 0.045407865196466446,
      "w_low_ratio": 0.020685997209511697,
      "w_max": 1.9191896319389343,
      "w_mean": 1.28102046251297,
      "w_min": 1.1079171471645686e-36,
      "w_std": 0.15209556370973587
    },
    {
      "completion_length": 3622.8438110351562,
      "cov_mean": -1.4215014289220562e-05,
      "cov_std": 0.19959762692451477,
      "entropy": 0.43701171875,
      "epoch": 0.03314285714285714,
      "grad_norm": 0.11793094128370285,
      "kl": 0.00043398141860961914,
      "learning_rate": 9.140576474687263e-07,
      "loss": 0.0686,
      "reward": 0.2812500149011612,
      "reward_std": 0.343124657869339,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.2187500074505806,
      "step": 29,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03135715611279011,
      "w_max": 1.389756977558136,
      "w_mean": 1.1044960916042328,
      "w_min": 0.5,
      "w_std": 0.15529824048280716
    },
    {
      "completion_length": 3211.6563110351562,
      "cov_mean": 2.0011442074974184e-05,
      "cov_std": 0.3438211902976036,
      "entropy": 0.40087890625,
      "epoch": 0.03428571428571429,
      "grad_norm": 0.15425726771354675,
      "kl": 0.0005748271942138672,
      "learning_rate": 9.046048391230247e-07,
      "loss": 0.096,
      "reward": 0.7708333544433117,
      "reward_std": 0.6762835085391998,
      "rewards/accuracy_reward": 0.2812500139698386,
      "rewards/format_reward": 0.489583358168602,
      "step": 30,
      "w_high_ratio": 0.0955454632639885,
      "w_low_ratio": 0.04459251323714852,
      "w_max": 1.7270594835281372,
      "w_mean": 1.2695180475711823,
      "w_min": 0.0,
      "w_std": 0.22970640659332275
    },
    {
      "completion_length": 3313.7500610351562,
      "cov_mean": -6.734976523148362e-07,
      "cov_std": 0.1561581064015627,
      "entropy": 0.38623046875,
      "epoch": 0.03542857142857143,
      "grad_norm": 0.12495917081832886,
      "kl": 0.00024247169494628906,
      "learning_rate": 8.9471999940354e-07,
      "loss": 0.0659,
      "reward": 0.47916669212281704,
      "reward_std": 0.3414399288594723,
      "rewards/accuracy_reward": 0.16666667722165585,
      "rewards/format_reward": 0.31250001303851604,
      "step": 31,
      "w_high_ratio": 0.12074629962444305,
      "w_low_ratio": 0.02205055020749569,
      "w_max": 1.9694485068321228,
      "w_mean": 1.323824942111969,
      "w_min": 3.479372530225627e-30,
      "w_std": 0.15388164669275284
    },
    {
      "completion_length": 3430.5521850585938,
      "cov_mean": -1.8880080915550934e-05,
      "cov_std": 0.24603740125894547,
      "entropy": 0.4443359375,
      "epoch": 0.036571428571428574,
      "grad_norm": 0.10446158051490784,
      "kl": 0.00040030479431152344,
      "learning_rate": 8.844151714648274e-07,
      "loss": 0.0417,
      "reward": 0.6875000223517418,
      "reward_std": 0.4970519095659256,
      "rewards/accuracy_reward": 0.2604166679084301,
      "rewards/format_reward": 0.4270833507180214,
      "step": 32,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03472677152603865,
      "w_max": 1.578925609588623,
      "w_mean": 1.1632550954818726,
      "w_min": 0.0,
      "w_std": 0.17994992434978485
    },
    {
      "completion_length": 3569.229248046875,
      "cov_mean": -1.2864127711509354e-05,
      "cov_std": 0.21655914932489395,
      "entropy": 0.3828125,
      "epoch": 0.037714285714285714,
      "grad_norm": 0.11952047049999237,
      "kl": 0.00048720836639404297,
      "learning_rate": 8.737029101523929e-07,
      "loss": 0.0213,
      "reward": 0.5833333507180214,
      "reward_std": 0.4569981172680855,
      "rewards/accuracy_reward": 0.229166679084301,
      "rewards/format_reward": 0.3541666716337204,
      "step": 33,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03175507392734289,
      "w_max": 1.3123357892036438,
      "w_mean": 1.0974721312522888,
      "w_min": 0.25,
      "w_std": 0.16161495074629784
    },
    {
      "completion_length": 2714.0000610351562,
      "cov_mean": -3.014505455212202e-05,
      "cov_std": 0.24434123933315277,
      "entropy": 0.462890625,
      "epoch": 0.038857142857142854,
      "grad_norm": 0.24054297804832458,
      "kl": 0.0010285377502441406,
      "learning_rate": 8.625962667065487e-07,
      "loss": 0.0264,
      "reward": 0.8541666865348816,
      "reward_std": 0.43565599620342255,
      "rewards/accuracy_reward": 0.3333333432674408,
      "rewards/format_reward": 0.5208333432674408,
      "step": 34,
      "w_high_ratio": 0.08968023210763931,
      "w_low_ratio": 0.03058682754635811,
      "w_max": 1.8369105458259583,
      "w_mean": 1.3255797028541565,
      "w_min": 0.25,
      "w_std": 0.23013706505298615
    },
    {
      "completion_length": 3206.260498046875,
      "cov_mean": 1.827982691793295e-05,
      "cov_std": 0.2361072190105915,
      "entropy": 0.42578125,
      "epoch": 0.04,
      "grad_norm": 0.13136403262615204,
      "kl": 0.0009332895278930664,
      "learning_rate": 8.511087728614862e-07,
      "loss": 0.0513,
      "reward": 0.5625000149011612,
      "reward_std": 0.47499874979257584,
      "rewards/accuracy_reward": 0.1979166679084301,
      "rewards/format_reward": 0.3645833469927311,
      "step": 35,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03539817640557885,
      "w_max": 1.4059478044509888,
      "w_mean": 1.1172049045562744,
      "w_min": 0.0,
      "w_std": 0.1749916821718216
    },
    {
      "completion_length": 3699.416748046875,
      "cov_mean": -1.4997711559772142e-05,
      "cov_std": 0.20064959302544594,
      "entropy": 0.50439453125,
      "epoch": 0.04114285714285714,
      "grad_norm": 0.1509845107793808,
      "kl": 0.0011619925498962402,
      "learning_rate": 8.392544243589427e-07,
      "loss": 0.0429,
      "reward": 0.2500000102445483,
      "reward_std": 0.41391417384147644,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.21875000279396772,
      "step": 36,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.029611330712214112,
      "w_max": 1.3176401853561401,
      "w_mean": 1.0739335417747498,
      "w_min": 0.0,
      "w_std": 0.15683909878134727
    },
    {
      "completion_length": 3516.1563110351562,
      "cov_mean": -2.547230405980372e-05,
      "cov_std": 0.11416707932949066,
      "entropy": 0.43994140625,
      "epoch": 0.04228571428571429,
      "grad_norm": 0.08760611712932587,
      "kl": 0.0007746219635009766,
      "learning_rate": 8.270476638965461e-07,
      "loss": 0.0156,
      "reward": 0.22916667442768812,
      "reward_std": 0.19299374520778656,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.21875001024454832,
      "step": 37,
      "w_high_ratio": 0.05747595056891441,
      "w_low_ratio": 0.013685875572264194,
      "w_max": 1.6239450573921204,
      "w_mean": 1.1775790452957153,
      "w_min": 0.25,
      "w_std": 0.12097344920039177
    },
    {
      "completion_length": 3670.822998046875,
      "cov_mean": -4.859739419771358e-06,
      "cov_std": 0.11942135915160179,
      "entropy": 0.4833984375,
      "epoch": 0.04342857142857143,
      "grad_norm": 0.08295677602291107,
      "kl": 0.0007152557373046875,
      "learning_rate": 8.145033635316128e-07,
      "loss": 0.016,
      "reward": 0.322916679084301,
      "reward_std": 0.24508872628211975,
      "rewards/accuracy_reward": 0.14583333395421505,
      "rewards/format_reward": 0.1770833432674408,
      "step": 38,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.015665842220187187,
      "w_max": 1.2278587818145752,
      "w_mean": 1.0692134499549866,
      "w_min": 0.5,
      "w_std": 0.08390428125858307
    },
    {
      "completion_length": 3133.5521850585938,
      "cov_mean": 1.6014040738809854e-05,
      "cov_std": 0.15454116463661194,
      "entropy": 0.38427734375,
      "epoch": 0.044571428571428574,
      "grad_norm": 0.09236446022987366,
      "kl": 0.0011830329895019531,
      "learning_rate": 8.01636806561836e-07,
      "loss": 0.0191,
      "reward": 0.770833358168602,
      "reward_std": 0.30482664704322815,
      "rewards/accuracy_reward": 0.25000000558793545,
      "rewards/format_reward": 0.520833358168602,
      "step": 39,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.021826621610671282,
      "w_max": 1.4440618753433228,
      "w_mean": 1.1500347554683685,
      "w_min": 0.25,
      "w_std": 0.10476426035165787
    },
    {
      "completion_length": 2921.5938110351562,
      "cov_mean": 3.307407860120293e-05,
      "cov_std": 0.18591826409101486,
      "entropy": 0.4111328125,
      "epoch": 0.045714285714285714,
      "grad_norm": 0.1281704157590866,
      "kl": 0.0041351318359375,
      "learning_rate": 7.884636689049422e-07,
      "loss": 0.0448,
      "reward": 0.6770833507180214,
      "reward_std": 0.39515648037195206,
      "rewards/accuracy_reward": 0.17708333395421505,
      "rewards/format_reward": 0.5000000149011612,
      "step": 40,
      "w_high_ratio": 0.09708013385534286,
      "w_low_ratio": 0.02820506482385099,
      "w_max": 1.9662592709064484,
      "w_mean": 1.3416504263877869,
      "w_min": 1.7285604841107016e-17,
      "w_std": 0.17729274183511734
    },
    {
      "completion_length": 3497.4063110351562,
      "cov_mean": -7.282104343175888e-05,
      "cov_std": 0.2843910865485668,
      "entropy": 0.40771484375,
      "epoch": 0.046857142857142854,
      "grad_norm": 0.16035234928131104,
      "kl": 0.0008625984191894531,
      "learning_rate": 7.75e-07,
      "loss": 0.0447,
      "reward": 0.4583333507180214,
      "reward_std": 0.5345464050769806,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.3333333507180214,
      "step": 41,
      "w_high_ratio": 0.015127741731703281,
      "w_low_ratio": 0.04166511259973049,
      "w_max": 1.6356081068515778,
      "w_mean": 1.1487390100955963,
      "w_min": 3.531651517161998e-24,
      "w_std": 0.21111097559332848
    },
    {
      "completion_length": 3070.854248046875,
      "cov_mean": 4.5878337004978675e-06,
      "cov_std": 0.0855317497625947,
      "entropy": 0.48388671875,
      "epoch": 0.048,
      "grad_norm": 0.06175260245800018,
      "kl": 0.0006914138793945312,
      "learning_rate": 7.612622032536507e-07,
      "loss": 0.0063,
      "reward": 0.3229166716337204,
      "reward_std": 0.17353228479623795,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.3125000074505806,
      "step": 42,
      "w_high_ratio": 0.05438845232129097,
      "w_low_ratio": 0.008411283954046667,
      "w_max": 1.544800043106079,
      "w_mean": 1.1694203615188599,
      "w_min": 0.5,
      "w_std": 0.07276808470487595
    },
    {
      "completion_length": 3378.8125610351562,
      "cov_mean": -2.5848277346085524e-05,
      "cov_std": 0.2625325694680214,
      "entropy": 0.43701171875,
      "epoch": 0.04914285714285714,
      "grad_norm": 0.18838584423065186,
      "kl": 0.0014505386352539062,
      "learning_rate": 7.472670160550848e-07,
      "loss": 0.0805,
      "reward": 0.479166679084301,
      "reward_std": 0.5196144729852676,
      "rewards/accuracy_reward": 0.18750000558793545,
      "rewards/format_reward": 0.2916666716337204,
      "step": 43,
      "w_high_ratio": 0.05749715492129326,
      "w_low_ratio": 0.038668573601171374,
      "w_max": 1.9480818212032318,
      "w_mean": 1.1805387139320374,
      "w_min": 2.3359345679368763e-34,
      "w_std": 0.1971494909375906
    },
    {
      "completion_length": 2916.9791717529297,
      "cov_mean": -3.640100658230949e-06,
      "cov_std": 0.23978274501860142,
      "entropy": 0.41162109375,
      "epoch": 0.05028571428571429,
      "grad_norm": 0.16967085003852844,
      "kl": 0.005632162094116211,
      "learning_rate": 7.330314893841101e-07,
      "loss": 0.0373,
      "reward": 0.7708333460614085,
      "reward_std": 0.43626825511455536,
      "rewards/accuracy_reward": 0.2812500074505806,
      "rewards/format_reward": 0.4895833386108279,
      "step": 44,
      "w_high_ratio": 0.11527429521083832,
      "w_low_ratio": 0.0326957437209785,
      "w_max": 1.8800793588161469,
      "w_mean": 1.356241375207901,
      "w_min": 0.0,
      "w_std": 0.22198213264346123
    },
    {
      "completion_length": 3686.1875610351562,
      "cov_mean": -1.2132580195611808e-05,
      "cov_std": 0.17040352895855904,
      "entropy": 0.4228515625,
      "epoch": 0.05142857142857143,
      "grad_norm": 0.0952582135796547,
      "kl": 0.0019674301147460938,
      "learning_rate": 7.185729670371604e-07,
      "loss": 0.0175,
      "reward": 0.4687500149011612,
      "reward_std": 0.3746139518916607,
      "rewards/accuracy_reward": 0.2187500074505806,
      "rewards/format_reward": 0.2500000074505806,
      "step": 45,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.022616846952587366,
      "w_max": 1.2881874740123749,
      "w_mean": 1.0595116317272186,
      "w_min": 0.25,
      "w_std": 0.11086289770901203
    },
    {
      "completion_length": 3573.8229370117188,
      "cov_mean": -5.21990023116814e-06,
      "cov_std": 0.09164197091013193,
      "entropy": 0.5361328125,
      "epoch": 0.052571428571428575,
      "grad_norm": 0.061532407999038696,
      "kl": 0.0024976730346679688,
      "learning_rate": 7.039090644965509e-07,
      "loss": 0.0226,
      "reward": 0.20833333488553762,
      "reward_std": 0.21344273164868355,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.1666666679084301,
      "step": 46,
      "w_high_ratio": 0.05877559259533882,
      "w_low_ratio": 0.011987740639597178,
      "w_max": 1.5505282580852509,
      "w_mean": 1.1429267823696136,
      "w_min": 3.571343117882909e-28,
      "w_std": 0.09135792590677738
    },
    {
      "completion_length": 3139.9375610351562,
      "cov_mean": 1.948155477293767e-05,
      "cov_std": 0.3308473080396652,
      "entropy": 0.43017578125,
      "epoch": 0.053714285714285714,
      "grad_norm": 0.287928968667984,
      "kl": 0.0011713504791259766,
      "learning_rate": 6.890576474687263e-07,
      "loss": 0.0536,
      "reward": 0.8541666716337204,
      "reward_std": 0.5766339302062988,
      "rewards/accuracy_reward": 0.322916679084301,
      "rewards/format_reward": 0.5312500149011612,
      "step": 47,
      "w_high_ratio": 0.12357743084430695,
      "w_low_ratio": 0.038592321798205376,
      "w_max": 2.0394512712955475,
      "w_mean": 1.3565464913845062,
      "w_min": 0.25,
      "w_std": 0.26638074964284897
    },
    {
      "completion_length": 3051.385467529297,
      "cov_mean": -6.733167197126022e-06,
      "cov_std": 0.19061635434627533,
      "entropy": 0.4501953125,
      "epoch": 0.054857142857142854,
      "grad_norm": 0.1374855488538742,
      "kl": 0.00525665283203125,
      "learning_rate": 6.740368101176495e-07,
      "loss": 0.0486,
      "reward": 0.5208333432674408,
      "reward_std": 0.37667082995176315,
      "rewards/accuracy_reward": 0.1770833395421505,
      "rewards/format_reward": 0.34375000838190317,
      "step": 48,
      "w_high_ratio": 0.12677159160375595,
      "w_low_ratio": 0.023622059728950262,
      "w_max": 2.024912714958191,
      "w_mean": 1.3519074320793152,
      "w_min": 7.271374424684445e-33,
      "w_std": 0.19886896945536137
    },
    {
      "completion_length": 2580.0521850585938,
      "cov_mean": 2.3781666641298216e-05,
      "cov_std": 0.2574050724506378,
      "entropy": 0.39306640625,
      "epoch": 0.056,
      "grad_norm": 0.13522955775260925,
      "kl": 0.0030527114868164062,
      "learning_rate": 6.588648530198504e-07,
      "loss": 0.0379,
      "reward": 0.8020833730697632,
      "reward_std": 0.47789302468299866,
      "rewards/accuracy_reward": 0.20833333395421505,
      "rewards/format_reward": 0.5937500149011612,
      "step": 49,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03599585313349962,
      "w_max": 1.5133522152900696,
      "w_mean": 1.177164077758789,
      "w_min": 3.952712643244228e-41,
      "w_std": 0.19642843678593636
    },
    {
      "completion_length": 3276.2188110351562,
      "cov_mean": 3.7818183841409336e-05,
      "cov_std": 0.1878571268171072,
      "entropy": 0.36767578125,
      "epoch": 0.05714285714285714,
      "grad_norm": 0.10174579173326492,
      "kl": 0.0021190643310546875,
      "learning_rate": 6.435602608679916e-07,
      "loss": 0.04,
      "reward": 0.5937500102445483,
      "reward_std": 0.3714478053152561,
      "rewards/accuracy_reward": 0.2604166716337204,
      "rewards/format_reward": 0.3333333386108279,
      "step": 50,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.0255408501252532,
      "w_max": 1.3753422796726227,
      "w_mean": 1.1391299068927765,
      "w_min": 0.0,
      "w_std": 0.12728617619723082
    },
    {
      "completion_length": 2626.7084350585938,
      "cov_mean": -8.886720934242476e-06,
      "cov_std": 0.154384421184659,
      "entropy": 0.46826171875,
      "epoch": 0.05828571428571429,
      "grad_norm": 0.10762708634138107,
      "kl": 0.006221771240234375,
      "learning_rate": 6.281416799501187e-07,
      "loss": 0.038,
      "reward": 0.5416666865348816,
      "reward_std": 0.27000611275434494,
      "rewards/accuracy_reward": 0.08333333674818277,
      "rewards/format_reward": 0.4583333358168602,
      "step": 51,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.022287086583673954,
      "w_max": 1.522942990064621,
      "w_mean": 1.16915962100029,
      "w_min": 0.25,
      "w_std": 0.10866253450512886
    },
    {
      "completion_length": 3225.6875610351562,
      "cov_mean": -3.420543362153694e-05,
      "cov_std": 0.3140456900000572,
      "entropy": 0.41357421875,
      "epoch": 0.05942857142857143,
      "grad_norm": 0.16037067770957947,
      "kl": 0.0017061233520507812,
      "learning_rate": 6.126278954320294e-07,
      "loss": 0.0307,
      "reward": 0.833333358168602,
      "reward_std": 0.5814172253012657,
      "rewards/accuracy_reward": 0.3854166865348816,
      "rewards/format_reward": 0.4479166716337204,
      "step": 52,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.041595788672566414,
      "w_max": 1.4235666990280151,
      "w_mean": 1.1377580165863037,
      "w_min": 0.25,
      "w_std": 0.20165112614631653
    },
    {
      "completion_length": 3038.3126220703125,
      "cov_mean": 3.059411119465949e-05,
      "cov_std": 0.34266950748860836,
      "entropy": 0.43994140625,
      "epoch": 0.060571428571428575,
      "grad_norm": 0.18063929677009583,
      "kl": 0.004637241363525391,
      "learning_rate": 5.97037808470444e-07,
      "loss": 0.0228,
      "reward": 0.9062500447034836,
      "reward_std": 0.5983624011278152,
      "rewards/accuracy_reward": 0.3229166828095913,
      "rewards/format_reward": 0.583333358168602,
      "step": 53,
      "w_high_ratio": 0.0893278568983078,
      "w_low_ratio": 0.048161128303036094,
      "w_max": 1.605346292257309,
      "w_mean": 1.2202682793140411,
      "w_min": 3.5021185811519566e-32,
      "w_std": 0.2196234930306673
    },
    {
      "completion_length": 3048.3021850585938,
      "cov_mean": -3.5217308322899044e-05,
      "cov_std": 0.40455804020166397,
      "entropy": 0.4111328125,
      "epoch": 0.061714285714285715,
      "grad_norm": 0.26634302735328674,
      "kl": 0.0014967918395996094,
      "learning_rate": 5.813904131848564e-07,
      "loss": 0.0485,
      "reward": 1.0104167088866234,
      "reward_std": 0.6799703985452652,
      "rewards/accuracy_reward": 0.4375000111758709,
      "rewards/format_reward": 0.5729166828095913,
      "step": 54,
      "w_high_ratio": 0.14361883699893951,
      "w_low_ratio": 0.03988745156675577,
      "w_max": 1.914384812116623,
      "w_mean": 1.3435330390930176,
      "w_min": 0.0,
      "w_std": 0.2648167684674263
    },
    {
      "completion_length": 3372.3959350585938,
      "cov_mean": 5.4979325341264484e-06,
      "cov_std": 0.25056118331849575,
      "entropy": 0.44189453125,
      "epoch": 0.06285714285714286,
      "grad_norm": 0.0977473184466362,
      "kl": 0.0019044876098632812,
      "learning_rate": 5.657047735161255e-07,
      "loss": 0.0381,
      "reward": 0.6562500223517418,
      "reward_std": 0.4964308738708496,
      "rewards/accuracy_reward": 0.27083334140479565,
      "rewards/format_reward": 0.3854166716337204,
      "step": 55,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.0342027700971812,
      "w_max": 1.4320927858352661,
      "w_mean": 1.1150383353233337,
      "w_min": 0.0,
      "w_std": 0.18786128982901573
    },
    {
      "completion_length": 3225.2709045410156,
      "cov_mean": 5.9108706409460865e-06,
      "cov_std": 0.22232061624526978,
      "entropy": 0.43115234375,
      "epoch": 0.064,
      "grad_norm": 0.11878591775894165,
      "kl": 0.00154876708984375,
      "learning_rate": 5.5e-07,
      "loss": 0.0112,
      "reward": 0.6458333656191826,
      "reward_std": 0.39009611308574677,
      "rewards/accuracy_reward": 0.19791666697710752,
      "rewards/format_reward": 0.4479166828095913,
      "step": 56,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.028656802838668227,
      "w_max": 1.5001116394996643,
      "w_mean": 1.2033225297927856,
      "w_min": 1.9247332911380988e-31,
      "w_std": 0.18417230807244778
    },
    {
      "completion_length": 3587.6458740234375,
      "cov_mean": -4.519502726907376e-05,
      "cov_std": 0.24361642450094223,
      "entropy": 0.34423828125,
      "epoch": 0.06514285714285714,
      "grad_norm": 0.11000871658325195,
      "kl": 0.0006794929504394531,
      "learning_rate": 5.342952264838747e-07,
      "loss": 0.0423,
      "reward": 0.4375,
      "reward_std": 0.467288788408041,
      "rewards/accuracy_reward": 0.1354166716337204,
      "rewards/format_reward": 0.3020833358168602,
      "step": 57,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03233239706605673,
      "w_max": 1.2505627870559692,
      "w_mean": 1.0757884085178375,
      "w_min": 0.25,
      "w_std": 0.14500370249152184
    },
    {
      "completion_length": 2626.947998046875,
      "cov_mean": -1.4388041108759353e-05,
      "cov_std": 0.2534067742526531,
      "entropy": 0.38427734375,
      "epoch": 0.06628571428571428,
      "grad_norm": 0.15062791109085083,
      "kl": 0.0038776397705078125,
      "learning_rate": 5.186095868151436e-07,
      "loss": 0.0665,
      "reward": 0.916666716337204,
      "reward_std": 0.43652553856372833,
      "rewards/accuracy_reward": 0.2812500009313226,
      "rewards/format_reward": 0.635416692122817,
      "step": 58,
      "w_high_ratio": 0.12001378461718559,
      "w_low_ratio": 0.03707017982378602,
      "w_max": 2.2393843233585358,
      "w_mean": 1.5285146832466125,
      "w_min": 2.338311714957214e-41,
      "w_std": 0.2577071785926819
    },
    {
      "completion_length": 3455.0208740234375,
      "cov_mean": -2.0748303086293163e-05,
      "cov_std": 0.2206678595393896,
      "entropy": 0.3994140625,
      "epoch": 0.06742857142857143,
      "grad_norm": 0.1635066419839859,
      "kl": 0.0013880729675292969,
      "learning_rate": 5.02962191529556e-07,
      "loss": 0.0447,
      "reward": 0.39583333395421505,
      "reward_std": 0.3752100467681885,
      "rewards/accuracy_reward": 0.14583333861082792,
      "rewards/format_reward": 0.2500000102445483,
      "step": 59,
      "w_high_ratio": 0.121368907392025,
      "w_low_ratio": 0.02756796986795962,
      "w_max": 1.8036501705646515,
      "w_mean": 1.2463297247886658,
      "w_min": 0.25,
      "w_std": 0.14911611750721931
    },
    {
      "completion_length": 3254.92724609375,
      "cov_mean": -1.7401176137354923e-05,
      "cov_std": 0.17664196342229843,
      "entropy": 0.396484375,
      "epoch": 0.06857142857142857,
      "grad_norm": 0.08288750052452087,
      "kl": 0.0025072097778320312,
      "learning_rate": 4.873721045679706e-07,
      "loss": 0.0281,
      "reward": 0.4583333432674408,
      "reward_std": 0.39030885696411133,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.34375000558793545,
      "step": 60,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.022112081991508603,
      "w_max": 1.5709031820297241,
      "w_mean": 1.1311749517917633,
      "w_min": 4.6449540846206874e-42,
      "w_std": 0.12603357434272766
    },
    {
      "completion_length": 3376.6771850585938,
      "cov_mean": 2.7930617193305807e-05,
      "cov_std": 0.22145200800150633,
      "entropy": 0.400390625,
      "epoch": 0.06971428571428571,
      "grad_norm": 0.2130555361509323,
      "kl": 0.0013718605041503906,
      "learning_rate": 4.7185832004988133e-07,
      "loss": 0.0505,
      "reward": 0.6666666669771075,
      "reward_std": 0.4248874858021736,
      "rewards/accuracy_reward": 0.1979166716337204,
      "rewards/format_reward": 0.4687500102445483,
      "step": 61,
      "w_high_ratio": 0.02656024508178234,
      "w_low_ratio": 0.02897683286573738,
      "w_max": 1.5726596117019653,
      "w_mean": 1.1504198908805847,
      "w_min": 0.25,
      "w_std": 0.15992471296340227
    },
    {
      "completion_length": 2804.3438110351562,
      "cov_mean": 5.110432311994373e-05,
      "cov_std": 0.3370564728975296,
      "entropy": 0.385986328125,
      "epoch": 0.07085714285714285,
      "grad_norm": 0.20217926800251007,
      "kl": 0.0056667327880859375,
      "learning_rate": 4.5643973913200837e-07,
      "loss": 0.0984,
      "reward": 0.854166679084301,
      "reward_std": 0.6372481435537338,
      "rewards/accuracy_reward": 0.29166667722165585,
      "rewards/format_reward": 0.5625000074505806,
      "step": 62,
      "w_high_ratio": 0.15436114370822906,
      "w_low_ratio": 0.04324930440634489,
      "w_max": 2.3450452983379364,
      "w_mean": 1.4747015237808228,
      "w_min": 2.7272514644043088e-36,
      "w_std": 0.31457675993442535
    },
    {
      "completion_length": 2748.1876220703125,
      "cov_mean": 9.652720564190531e-06,
      "cov_std": 0.3283480554819107,
      "entropy": 0.43310546875,
      "epoch": 0.072,
      "grad_norm": 0.15674079954624176,
      "kl": 0.0032906532287597656,
      "learning_rate": 4.4113514698014953e-07,
      "loss": 0.0725,
      "reward": 1.031250037252903,
      "reward_std": 0.5360563546419144,
      "rewards/accuracy_reward": 0.3541666753590107,
      "rewards/format_reward": 0.6770833432674408,
      "step": 63,
      "w_high_ratio": 0.0625,
      "w_low_ratio": 0.04385069524869323,
      "w_max": 1.8759834170341492,
      "w_mean": 1.3464274108409882,
      "w_min": 0.0,
      "w_std": 0.1958361305296421
    },
    {
      "completion_length": 3358.3334350585938,
      "cov_mean": -4.510660664891475e-05,
      "cov_std": 0.2794957533478737,
      "entropy": 0.44921875,
      "epoch": 0.07314285714285715,
      "grad_norm": 0.12678052484989166,
      "kl": 0.005690097808837891,
      "learning_rate": 4.2596318988235037e-07,
      "loss": 0.0603,
      "reward": 0.5937500149011612,
      "reward_std": 0.583733007311821,
      "rewards/accuracy_reward": 0.2187500037252903,
      "rewards/format_reward": 0.3750000074505806,
      "step": 64,
      "w_high_ratio": 0.044796403497457504,
      "w_low_ratio": 0.03829633165150881,
      "w_max": 1.6352408528327942,
      "w_mean": 1.167921930551529,
      "w_min": 2.6764800668604006e-43,
      "w_std": 0.20047394558787346
    },
    {
      "completion_length": 3026.5209350585938,
      "cov_mean": 1.992606485146098e-05,
      "cov_std": 0.17664698883891106,
      "entropy": 0.38525390625,
      "epoch": 0.07428571428571429,
      "grad_norm": 0.08567796647548676,
      "kl": 0.0034112930297851562,
      "learning_rate": 4.1094235253127374e-07,
      "loss": 0.0309,
      "reward": 0.6250000260770321,
      "reward_std": 0.32293669879436493,
      "rewards/accuracy_reward": 0.16666666697710752,
      "rewards/format_reward": 0.4583333395421505,
      "step": 65,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.029133206233382225,
      "w_max": 1.3120096027851105,
      "w_mean": 1.1221435964107513,
      "w_min": 1.0468715588988584e-22,
      "w_std": 0.12845914252102375
    },
    {
      "completion_length": 2413.1250610351562,
      "cov_mean": -4.743512135974015e-06,
      "cov_std": 0.05960770323872566,
      "entropy": 0.35693359375,
      "epoch": 0.07542857142857143,
      "grad_norm": 0.08712891489267349,
      "kl": 0.0032825469970703125,
      "learning_rate": 3.9609093550344907e-07,
      "loss": 0.0194,
      "reward": 0.8645833432674408,
      "reward_std": 0.13795074447989464,
      "rewards/accuracy_reward": 0.375,
      "rewards/format_reward": 0.4895833358168602,
      "step": 66,
      "w_high_ratio": 0.04171403869986534,
      "w_low_ratio": 0.006300564622506499,
      "w_max": 1.5700030624866486,
      "w_mean": 1.1926406025886536,
      "w_min": 0.5264238715171814,
      "w_std": 0.06322706118226051
    },
    {
      "completion_length": 3792.9271240234375,
      "cov_mean": -9.492634717389592e-06,
      "cov_std": 0.12886795960366726,
      "entropy": 0.36865234375,
      "epoch": 0.07657142857142857,
      "grad_norm": 0.0828777328133583,
      "kl": 0.002822399139404297,
      "learning_rate": 3.8142703296283953e-07,
      "loss": 0.0179,
      "reward": 0.21875000558793545,
      "reward_std": 0.2869785502552986,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.16666667442768812,
      "step": 67,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.017554222606122494,
      "w_max": 1.1846114993095398,
      "w_mean": 1.0396882444620132,
      "w_min": 0.25,
      "w_std": 0.09070100169628859
    },
    {
      "completion_length": 2557.6146240234375,
      "cov_mean": 1.5981570413714508e-05,
      "cov_std": 0.31408151611685753,
      "entropy": 0.44384765625,
      "epoch": 0.07771428571428571,
      "grad_norm": 0.20466886460781097,
      "kl": 0.0033721923828125,
      "learning_rate": 3.6696851061588994e-07,
      "loss": 0.0566,
      "reward": 0.8020833507180214,
      "reward_std": 0.5168246552348137,
      "rewards/accuracy_reward": 0.2395833432674408,
      "rewards/format_reward": 0.5625000149011612,
      "step": 68,
      "w_high_ratio": 0.04399501532316208,
      "w_low_ratio": 0.042061637388542295,
      "w_max": 1.8962246477603912,
      "w_mean": 1.305674433708191,
      "w_min": 4.959685019058809e-39,
      "w_std": 0.2475343719124794
    },
    {
      "completion_length": 3009.7084350585938,
      "cov_mean": -4.52714293714962e-05,
      "cov_std": 0.23395150154829025,
      "entropy": 0.5517578125,
      "epoch": 0.07885714285714286,
      "grad_norm": 0.18218518793582916,
      "kl": 0.014312744140625,
      "learning_rate": 3.5273298394491515e-07,
      "loss": 0.0752,
      "reward": 0.4687500149011612,
      "reward_std": 0.43293242901563644,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.37500000558793545,
      "step": 69,
      "w_high_ratio": 0.057301439344882965,
      "w_low_ratio": 0.0333517212420702,
      "w_max": 1.9855602085590363,
      "w_mean": 1.2992196083068848,
      "w_min": 7.707141553786494e-45,
      "w_std": 0.19260139763355255
    },
    {
      "completion_length": 3280.0833740234375,
      "cov_mean": 2.8561088129208656e-05,
      "cov_std": 0.19498306885361671,
      "entropy": 0.3720703125,
      "epoch": 0.08,
      "grad_norm": 0.10543849319219589,
      "kl": 0.010352134704589844,
      "learning_rate": 3.387377967463493e-07,
      "loss": 0.0185,
      "reward": 0.5416667014360428,
      "reward_std": 0.3840207904577255,
      "rewards/accuracy_reward": 0.1250000074505806,
      "rewards/format_reward": 0.4166666716337204,
      "step": 70,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.02808787301182747,
      "w_max": 1.2882064878940582,
      "w_mean": 1.0968604385852814,
      "w_min": 0.25,
      "w_std": 0.1326066516339779
    },
    {
      "completion_length": 2855.7708740234375,
      "cov_mean": 4.004434208582097e-05,
      "cov_std": 0.13545112498104572,
      "entropy": 0.42724609375,
      "epoch": 0.08114285714285714,
      "grad_norm": 0.11269883066415787,
      "kl": 0.014951705932617188,
      "learning_rate": 3.250000000000001e-07,
      "loss": 0.0154,
      "reward": 0.5729166967794299,
      "reward_std": 0.2492993399500847,
      "rewards/accuracy_reward": 0.1979166716337204,
      "rewards/format_reward": 0.3750000027939677,
      "step": 71,
      "w_high_ratio": 0.08807118237018585,
      "w_low_ratio": 0.01654834917280823,
      "w_max": 1.8029770255088806,
      "w_mean": 1.2483810186386108,
      "w_min": 0.25,
      "w_std": 0.1558239422738552
    },
    {
      "completion_length": 3550.322998046875,
      "cov_mean": -4.4116359276813455e-07,
      "cov_std": 0.24292385205626488,
      "entropy": 0.5107421875,
      "epoch": 0.08228571428571428,
      "grad_norm": 0.20913389325141907,
      "kl": 0.0040874481201171875,
      "learning_rate": 3.115363310950578e-07,
      "loss": 0.0498,
      "reward": 0.3750000102445483,
      "reward_std": 0.40810926631093025,
      "rewards/accuracy_reward": 0.0729166716337204,
      "rewards/format_reward": 0.3020833386108279,
      "step": 72,
      "w_high_ratio": 0.033293891698122025,
      "w_low_ratio": 0.03450615704059601,
      "w_max": 1.6246004700660706,
      "w_mean": 1.1392557322978973,
      "w_min": 2.138569791073276e-36,
      "w_std": 0.19269496202468872
    },
    {
      "completion_length": 3837.1146240234375,
      "cov_mean": 2.1203804863034748e-06,
      "cov_std": 0.15915799140930176,
      "entropy": 0.51171875,
      "epoch": 0.08342857142857144,
      "grad_norm": 0.08118956536054611,
      "kl": 0.0014677047729492188,
      "learning_rate": 2.9836319343816397e-07,
      "loss": 0.0146,
      "reward": 0.2708333432674408,
      "reward_std": 0.25903886556625366,
      "rewards/accuracy_reward": 0.1041666716337204,
      "rewards/format_reward": 0.1666666716337204,
      "step": 73,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.020282023586332798,
      "w_max": 1.1690112948417664,
      "w_mean": 1.027027040719986,
      "w_min": 0.5,
      "w_std": 0.09119972214102745
    },
    {
      "completion_length": 3503.1250610351562,
      "cov_mean": -4.496445217228029e-05,
      "cov_std": 0.2548239603638649,
      "entropy": 0.41748046875,
      "epoch": 0.08457142857142858,
      "grad_norm": 0.1568731665611267,
      "kl": 0.00296783447265625,
      "learning_rate": 2.854966364683872e-07,
      "loss": 0.0655,
      "reward": 0.5625000074505806,
      "reward_std": 0.4641239196062088,
      "rewards/accuracy_reward": 0.2604166679084301,
      "rewards/format_reward": 0.3020833358168602,
      "step": 74,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.033767144195735455,
      "w_max": 1.4706333875656128,
      "w_mean": 1.078809916973114,
      "w_min": 0.25,
      "w_std": 0.1507711410522461
    },
    {
      "completion_length": 3346.1875610351562,
      "cov_mean": -7.328241736104246e-06,
      "cov_std": 0.20871411636471748,
      "entropy": 0.404296875,
      "epoch": 0.08571428571428572,
      "grad_norm": 0.10099554806947708,
      "kl": 0.0059261322021484375,
      "learning_rate": 2.729523361034538e-07,
      "loss": 0.0554,
      "reward": 0.572916679084301,
      "reward_std": 0.3865407630801201,
      "rewards/accuracy_reward": 0.1875,
      "rewards/format_reward": 0.3854166716337204,
      "step": 75,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.032186293974518776,
      "w_max": 1.5220047235488892,
      "w_mean": 1.1460089683532715,
      "w_min": 0.25,
      "w_std": 0.17850109934806824
    },
    {
      "completion_length": 3092.635498046875,
      "cov_mean": 4.942317445966182e-06,
      "cov_std": 0.20054961927235126,
      "entropy": 0.44091796875,
      "epoch": 0.08685714285714285,
      "grad_norm": 0.14435574412345886,
      "kl": 0.001827239990234375,
      "learning_rate": 2.6074557564105724e-07,
      "loss": 0.02,
      "reward": 0.583333358168602,
      "reward_std": 0.34913603961467743,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.4791666939854622,
      "step": 76,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.030099061783403158,
      "w_max": 1.574487328529358,
      "w_mean": 1.184053212404251,
      "w_min": 1.2891045632755887e-30,
      "w_std": 0.1306541245430708
    },
    {
      "completion_length": 3414.0938110351562,
      "cov_mean": 4.712834697784274e-05,
      "cov_std": 0.23044732213020325,
      "entropy": 0.45263671875,
      "epoch": 0.088,
      "grad_norm": 0.27278050780296326,
      "kl": 0.0013208389282226562,
      "learning_rate": 2.488912271385139e-07,
      "loss": 0.015,
      "reward": 0.510416672565043,
      "reward_std": 0.4236603006720543,
      "rewards/accuracy_reward": 0.1354166716337204,
      "rewards/format_reward": 0.37500001583248377,
      "step": 77,
      "w_high_ratio": 0.006981382612138987,
      "w_low_ratio": 0.02834776253439486,
      "w_max": 1.5653101801872253,
      "w_mean": 1.1680251359939575,
      "w_min": 5.385388925526598e-27,
      "w_std": 0.172462142072618
    },
    {
      "completion_length": 3528.5001220703125,
      "cov_mean": -2.2849541437608423e-05,
      "cov_std": 0.24372886680066586,
      "entropy": 0.41015625,
      "epoch": 0.08914285714285715,
      "grad_norm": 0.13359344005584717,
      "kl": 0.0022611618041992188,
      "learning_rate": 2.374037332934512e-07,
      "loss": 0.037,
      "reward": 0.6562500186264515,
      "reward_std": 0.5271749570965767,
      "rewards/accuracy_reward": 0.2812500111758709,
      "rewards/format_reward": 0.3750000111758709,
      "step": 78,
      "w_high_ratio": 0.041247133165597916,
      "w_low_ratio": 0.027018944965675473,
      "w_max": 1.6266585290431976,
      "w_mean": 1.1576823890209198,
      "w_min": 1.890902738208876e-34,
      "w_std": 0.16164034884423018
    },
    {
      "completion_length": 2676.187545776367,
      "cov_mean": 1.6489982044731732e-05,
      "cov_std": 0.21785889007151127,
      "entropy": 0.34912109375,
      "epoch": 0.09028571428571429,
      "grad_norm": 0.1319928616285324,
      "kl": 0.0022878646850585938,
      "learning_rate": 2.2629708984760706e-07,
      "loss": 0.0289,
      "reward": 0.843750037252903,
      "reward_std": 0.4268086552619934,
      "rewards/accuracy_reward": 0.26041667349636555,
      "rewards/format_reward": 0.5833333544433117,
      "step": 79,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03145516477525234,
      "w_max": 1.4992458820343018,
      "w_mean": 1.168209046125412,
      "w_min": 0.0,
      "w_std": 0.1453277636319399
    },
    {
      "completion_length": 3573.8021240234375,
      "cov_mean": -9.463059541303664e-06,
      "cov_std": 0.1826024018228054,
      "entropy": 0.48974609375,
      "epoch": 0.09142857142857143,
      "grad_norm": 0.0996597409248352,
      "kl": 0.002468109130859375,
      "learning_rate": 2.1558482853517253e-07,
      "loss": 0.0271,
      "reward": 0.520833358168602,
      "reward_std": 0.3810138627886772,
      "rewards/accuracy_reward": 0.1979166679084301,
      "rewards/format_reward": 0.3229166753590107,
      "step": 80,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.02435835381038487,
      "w_max": 1.368900626897812,
      "w_mean": 1.106232464313507,
      "w_min": 0.25,
      "w_std": 0.11319147422909737
    },
    {
      "completion_length": 3308.625,
      "cov_mean": -1.9006092770723626e-05,
      "cov_std": 0.1842699982225895,
      "entropy": 0.58740234375,
      "epoch": 0.09257142857142857,
      "grad_norm": 0.15561415255069733,
      "kl": 0.005096435546875,
      "learning_rate": 2.0528000059645995e-07,
      "loss": 0.0328,
      "reward": 0.4062500149011612,
      "reward_std": 0.28905032202601433,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.291666679084301,
      "step": 81,
      "w_high_ratio": 0.125,
      "w_low_ratio": 0.02731443475931883,
      "w_max": 1.497319370508194,
      "w_mean": 1.1954041719436646,
      "w_min": 0.25,
      "w_std": 0.12777045369148254
    },
    {
      "completion_length": 3054.5938110351562,
      "cov_mean": -1.190575176224229e-05,
      "cov_std": 0.13467486761510372,
      "entropy": 0.453125,
      "epoch": 0.09371428571428571,
      "grad_norm": 0.06870616227388382,
      "kl": 0.0037078857421875,
      "learning_rate": 1.9539516087697517e-07,
      "loss": 0.0283,
      "reward": 0.6458333432674408,
      "reward_std": 0.2705298960208893,
      "rewards/accuracy_reward": 0.2187500074505806,
      "rewards/format_reward": 0.4270833432674408,
      "step": 82,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.016570631880313158,
      "w_max": 1.6605907380580902,
      "w_mean": 1.2332959175109863,
      "w_min": 0.25,
      "w_std": 0.10085548926144838
    },
    {
      "completion_length": 3103.7188110351562,
      "cov_mean": -1.1289954500171007e-05,
      "cov_std": 0.25786374136805534,
      "entropy": 0.4951171875,
      "epoch": 0.09485714285714286,
      "grad_norm": 0.11641041934490204,
      "kl": 0.0033721923828125,
      "learning_rate": 1.8594235253127372e-07,
      "loss": 0.0561,
      "reward": 0.541666679084301,
      "reward_std": 0.5305610671639442,
      "rewards/accuracy_reward": 0.17708333488553762,
      "rewards/format_reward": 0.3645833507180214,
      "step": 83,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03598734503611922,
      "w_max": 1.3528369665145874,
      "w_mean": 1.095271646976471,
      "w_min": 0.0,
      "w_std": 0.1604925710707903
    },
    {
      "completion_length": 3278.104248046875,
      "cov_mean": 3.2929374356172048e-06,
      "cov_std": 0.246146522462368,
      "entropy": 0.45849609375,
      "epoch": 0.096,
      "grad_norm": 0.15934637188911438,
      "kl": 0.0013265609741210938,
      "learning_rate": 1.7693309235023127e-07,
      "loss": 0.0172,
      "reward": 0.7187500298023224,
      "reward_std": 0.432245634496212,
      "rewards/accuracy_reward": 0.3125000149011612,
      "rewards/format_reward": 0.4062500149011612,
      "step": 84,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.02858129981905222,
      "w_max": 1.45015150308609,
      "w_mean": 1.134983777999878,
      "w_min": 0.25,
      "w_std": 0.15726573020219803
    },
    {
      "completion_length": 3484.729248046875,
      "cov_mean": -8.481749773636693e-06,
      "cov_std": 0.29308537393808365,
      "entropy": 0.37841796875,
      "epoch": 0.09714285714285714,
      "grad_norm": 0.1379634290933609,
      "kl": 0.0017528533935546875,
      "learning_rate": 1.6837835672960831e-07,
      "loss": 0.0623,
      "reward": 0.5104166716337204,
      "reward_std": 0.6195737272500992,
      "rewards/accuracy_reward": 0.1666666679084301,
      "rewards/format_reward": 0.3437500037252903,
      "step": 85,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.03877481259405613,
      "w_max": 1.4117690026760101,
      "w_mean": 1.119108110666275,
      "w_min": 0.0,
      "w_std": 0.18827635422348976
    },
    {
      "completion_length": 3158.8125,
      "cov_mean": 6.843166598713424e-06,
      "cov_std": 0.12266075890511274,
      "entropy": 0.47119140625,
      "epoch": 0.09828571428571428,
      "grad_norm": 0.09657198935747147,
      "kl": 0.0029735565185546875,
      "learning_rate": 1.6028856829700258e-07,
      "loss": 0.0174,
      "reward": 0.541666679084301,
      "reward_std": 0.2581377625465393,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.4166666716337204,
      "step": 86,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.018843807047232985,
      "w_max": 1.3829069435596466,
      "w_mean": 1.131670981645584,
      "w_min": 0.25,
      "w_std": 0.11318285018205643
    },
    {
      "completion_length": 3175.2188720703125,
      "cov_mean": -5.853003926858946e-05,
      "cov_std": 0.30259813368320465,
      "entropy": 0.556640625,
      "epoch": 0.09942857142857142,
      "grad_norm": 0.38111400604248047,
      "kl": 0.007335662841796875,
      "learning_rate": 1.5267358321348285e-07,
      "loss": 0.0778,
      "reward": 0.7187500149011612,
      "reward_std": 0.5092682540416718,
      "rewards/accuracy_reward": 0.2291666753590107,
      "rewards/format_reward": 0.4895833432674408,
      "step": 87,
      "w_high_ratio": 0.11183382570743561,
      "w_low_ratio": 0.03777051903307438,
      "w_max": 1.9268704950809479,
      "w_mean": 1.334629088640213,
      "w_min": 5.605193857299268e-45,
      "w_std": 0.27019689977169037
    },
    {
      "completion_length": 3190.9375610351562,
      "cov_mean": 7.67477886256529e-05,
      "cov_std": 0.46318161487579346,
      "entropy": 0.5048828125,
      "epoch": 0.10057142857142858,
      "grad_norm": 0.2448384165763855,
      "kl": 0.020694732666015625,
      "learning_rate": 1.4554267916537495e-07,
      "loss": 0.0906,
      "reward": 0.7604167014360428,
      "reward_std": 0.7656450867652893,
      "rewards/accuracy_reward": 0.291666679084301,
      "rewards/format_reward": 0.4687500149011612,
      "step": 88,
      "w_high_ratio": 0.08163053542375565,
      "w_low_ratio": 0.06242929771542549,
      "w_max": 1.8597923815250397,
      "w_mean": 1.2850928604602814,
      "w_min": 2.421886516239847e-28,
      "w_std": 0.3251073509454727
    },
    {
      "completion_length": 3597.2396850585938,
      "cov_mean": -4.437502229848178e-05,
      "cov_std": 0.29206302016973495,
      "entropy": 0.45703125,
      "epoch": 0.10171428571428572,
      "grad_norm": 0.16698718070983887,
      "kl": 0.00389862060546875,
      "learning_rate": 1.3890454406082956e-07,
      "loss": 0.0647,
      "reward": 0.4687500149011612,
      "reward_std": 0.5414880514144897,
      "rewards/accuracy_reward": 0.1770833395421505,
      "rewards/format_reward": 0.291666679084301,
      "step": 89,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.04101241147145629,
      "w_max": 1.4729963839054108,
      "w_mean": 1.0995305478572845,
      "w_min": 0.0,
      "w_std": 0.19675205275416374
    },
    {
      "completion_length": 2862.125,
      "cov_mean": -5.355579560273327e-06,
      "cov_std": 0.0925431028008461,
      "entropy": 0.638671875,
      "epoch": 0.10285714285714286,
      "grad_norm": 0.09346118569374084,
      "kl": 0.01725006103515625,
      "learning_rate": 1.3276726544494571e-07,
      "loss": 0.0269,
      "reward": 0.4791666716337204,
      "reward_std": 0.19776283204555511,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.4583333432674408,
      "step": 90,
      "w_high_ratio": 0.12039810419082642,
      "w_low_ratio": 0.014701983891427517,
      "w_max": 2.006953328847885,
      "w_mean": 1.3530822694301605,
      "w_min": 0.5,
      "w_std": 0.09125572815537453
    },
    {
      "completion_length": 3527.2813110351562,
      "cov_mean": -6.6539573708723765e-06,
      "cov_std": 0.19490721449255943,
      "entropy": 0.47802734375,
      "epoch": 0.104,
      "grad_norm": 0.16878993809223175,
      "kl": 0.0045948028564453125,
      "learning_rate": 1.2713832064634125e-07,
      "loss": 0.0023,
      "reward": 0.48958333395421505,
      "reward_std": 0.31570227444171906,
      "rewards/accuracy_reward": 0.17708333861082792,
      "rewards/format_reward": 0.3125000027939677,
      "step": 91,
      "w_high_ratio": 0.0432400144636631,
      "w_low_ratio": 0.025078749749809504,
      "w_max": 1.4252241849899292,
      "w_mean": 1.1511092782020569,
      "w_min": 0.25,
      "w_std": 0.14859570004045963
    },
    {
      "completion_length": 2996.8333740234375,
      "cov_mean": 5.000362762075383e-06,
      "cov_std": 0.274563018232584,
      "entropy": 0.44921875,
      "epoch": 0.10514285714285715,
      "grad_norm": 0.2672693729400635,
      "kl": 0.011915206909179688,
      "learning_rate": 1.220245676671809e-07,
      "loss": 0.0151,
      "reward": 0.6875000223517418,
      "reward_std": 0.3997742757201195,
      "rewards/accuracy_reward": 0.1562500074505806,
      "rewards/format_reward": 0.5312500074505806,
      "step": 92,
      "w_high_ratio": 0.057363301515579224,
      "w_low_ratio": 0.03872442920692265,
      "w_max": 1.9733782410621643,
      "w_mean": 1.2820636332035065,
      "w_min": 1.1237891646640876e-26,
      "w_std": 0.21614115312695503
    },
    {
      "completion_length": 3868.3333740234375,
      "cov_mean": 2.6024475801023073e-05,
      "cov_std": 0.11580366268754005,
      "entropy": 0.59130859375,
      "epoch": 0.10628571428571429,
      "grad_norm": 0.07178976386785507,
      "kl": 0.00457763671875,
      "learning_rate": 1.1743223682775649e-07,
      "loss": 0.0217,
      "reward": 0.10416666883975267,
      "reward_std": 0.23858631029725075,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.08333333488553762,
      "step": 93,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.022031503496691585,
      "w_max": 1.2298710346221924,
      "w_mean": 1.0156493484973907,
      "w_min": 9.954022423630139e-25,
      "w_std": 0.08076347131282091
    },
    {
      "completion_length": 3368.0834350585938,
      "cov_mean": -9.980105346585333e-07,
      "cov_std": 0.14382942207157612,
      "entropy": 0.56640625,
      "epoch": 0.10742857142857143,
      "grad_norm": 0.09354749321937561,
      "kl": 0.010030746459960938,
      "learning_rate": 1.1336692317580158e-07,
      "loss": 0.0167,
      "reward": 0.479166679084301,
      "reward_std": 0.26436545327305794,
      "rewards/accuracy_reward": 0.15625000651925802,
      "rewards/format_reward": 0.3229166679084301,
      "step": 94,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.020133810699917376,
      "w_max": 1.4812421798706055,
      "w_mean": 1.140279084444046,
      "w_min": 2.8643974210955766e-17,
      "w_std": 0.10274781100451946
    },
    {
      "completion_length": 3727.229248046875,
      "cov_mean": 2.8429121812223457e-07,
      "cov_std": 0.1946401260793209,
      "entropy": 0.4609375,
      "epoch": 0.10857142857142857,
      "grad_norm": 0.09702739864587784,
      "kl": 0.0018458366394042969,
      "learning_rate": 1.0983357966978745e-07,
      "loss": 0.039,
      "reward": 0.2916666707023978,
      "reward_std": 0.4342379942536354,
      "rewards/accuracy_reward": 0.07291666977107525,
      "rewards/format_reward": 0.21875000838190317,
      "step": 95,
      "w_high_ratio": 0.0,
      "w_low_ratio": 0.027697827550582588,
      "w_max": 1.2496315836906433,
      "w_mean": 1.0440161526203156,
      "w_min": 0.0,
      "w_std": 0.13094050344079733
    },
    {
      "completion_length": 3174.6876220703125,
      "cov_mean": 0.00011378643011994427,
      "cov_std": 0.20376956462860107,
      "entropy": 0.4482421875,
      "epoch": 0.10971428571428571,
      "grad_norm": 0.13061358034610748,
      "kl": 0.005021095275878906,
      "learning_rate": 1.068365111445064e-07,
      "loss": 0.007,
      "reward": 0.6562500074505806,
      "reward_std": 0.3372773453593254,
      "rewards/accuracy_reward": 0.2812500037252903,
      "rewards/format_reward": 0.3750000074505806,
      "step": 96,
      "w_high_ratio": 0.08483665436506271,
      "w_low_ratio": 0.023257225286215544,
      "w_max": 1.6405883729457855,
      "w_mean": 1.170585960149765,
      "w_min": 0.25,
      "w_std": 0.1635773852467537
    },
    {
      "completion_length": 3578.6458740234375,
      "cov_mean": 5.654522146869567e-05,
      "cov_std": 0.2582091810181737,
      "entropy": 0.4716796875,
      "epoch": 0.11085714285714286,
      "grad_norm": 0.23674072325229645,
      "kl": 0.0026683807373046875,
      "learning_rate": 1.0437936906629334e-07,
      "loss": 0.0282,
      "reward": 0.48958334419876337,
      "reward_std": 0.4029072895646095,
      "rewards/accuracy_reward": 0.21875000558793545,
      "rewards/format_reward": 0.27083334140479565,
      "step": 97,
      "w_high_ratio": 0.0427275113761425,
      "w_low_ratio": 0.02712295390665531,
      "w_max": 1.5456224977970123,
      "w_mean": 1.134882390499115,
      "w_min": 8.233329127140463e-42,
      "w_std": 0.17803996708244085
    },
    {
      "completion_length": 3275.70849609375,
      "cov_mean": -2.5506165911792777e-05,
      "cov_std": 0.22730276361107826,
      "entropy": 0.43359375,
      "epoch": 0.112,
      "grad_norm": 0.154100239276886,
      "kl": 0.0016241073608398438,
      "learning_rate": 1.0246514708427701e-07,
      "loss": 0.0791,
      "reward": 0.5208333432674408,
      "reward_std": 0.45262154936790466,
      "rewards/accuracy_reward": 0.14583333674818277,
      "rewards/format_reward": 0.3750000149011612,
      "step": 98,
      "w_high_ratio": 0.031526632606983185,
      "w_low_ratio": 0.030860408674925566,
      "w_max": 1.7042989134788513,
      "w_mean": 1.219407707452774,
      "w_min": 1.0509738482436128e-44,
      "w_std": 0.1900232806801796
    },
    {
      "completion_length": 3084.593795776367,
      "cov_mean": -6.495133902717498e-06,
      "cov_std": 0.13577165454626083,
      "entropy": 0.386474609375,
      "epoch": 0.11314285714285714,
      "grad_norm": 0.07463299483060837,
      "kl": 0.008008956909179688,
      "learning_rate": 1.0109617738307911e-07,
      "loss": 0.0185,
      "reward": 0.5208333488553762,
      "reward_std": 0.2819661721587181,
      "rewards/accuracy_reward": 0.19791667815297842,
      "rewards/format_reward": 0.3229166669771075,
      "step": 99,
      "w_high_ratio": 0.04817802831530571,
      "w_low_ratio": 0.017340978607535362,
      "w_max": 1.389538049697876,
      "w_mean": 1.1735607981681824,
      "w_min": 0.25,
      "w_std": 0.1109000938013196
    },
    {
      "completion_length": 3224.5834350585938,
      "cov_mean": -4.5620060973305954e-05,
      "cov_std": 0.26372817903757095,
      "entropy": 0.42724609375,
      "epoch": 0.11428571428571428,
      "grad_norm": 0.13250450789928436,
      "kl": 0.009317398071289062,
      "learning_rate": 1.002741278414069e-07,
      "loss": 0.0352,
      "reward": 0.7083333656191826,
      "reward_std": 0.4755344055593014,
      "rewards/accuracy_reward": 0.260416679084301,
      "rewards/format_reward": 0.447916679084301,
      "step": 100,
      "w_high_ratio": 0.05065765231847763,
      "w_low_ratio": 0.03619965072721243,
      "w_max": 1.6908635199069977,
      "w_mean": 1.1874340772628784,
      "w_min": 1.2465886678904214e-38,
      "w_std": 0.2075340449810028
    },
    {
      "epoch": 0.11428571428571428,
      "step": 100,
      "total_flos": 0.0,
      "train_loss": 0.041623960277065636,
      "train_runtime": 8415.8875,
      "train_samples_per_second": 1.141,
      "train_steps_per_second": 0.012
    }
  ],
  "logging_steps": 1,
  "max_steps": 100,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 6,
  "trial_name": null,
  "trial_params": null
}