{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.006,
  "eval_steps": 500,
  "global_step": 75,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2783.0,
      "completions/max_terminated_length": 2783.0,
      "completions/mean_length": 2052.75,
      "completions/mean_terminated_length": 2052.75,
      "completions/min_length": 1055.0,
      "completions/min_terminated_length": 1055.0,
      "entropy": 0.4163087382912636,
      "epoch": 8e-05,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.8663769960403442,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": -0.0386,
      "num_tokens": 78630.0,
      "reward": 0.46406251192092896,
      "reward_std": 0.20054946839809418,
      "rewards/rollout_reward_func/mean": 0.46406251192092896,
      "rewards/rollout_reward_func/std": 0.37604784965515137,
      "sampling/importance_sampling_ratio/max": 2.1498024463653564,
      "sampling/importance_sampling_ratio/mean": 1.0975958108901978,
      "sampling/importance_sampling_ratio/min": 0.241215318441391,
      "sampling/sampling_logp_difference/max": 0.7405228614807129,
      "sampling/sampling_logp_difference/mean": 0.039819031953811646,
      "step": 1,
      "step_time": 14.418279634999976
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2798.0,
      "completions/max_terminated_length": 2798.0,
      "completions/mean_length": 2084.21875,
      "completions/mean_terminated_length": 2084.21875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.3995310440659523,
      "epoch": 0.00016,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 2.142817735671997,
      "kl": 0.0,
      "learning_rate": 1.7142857142857143e-07,
      "loss": 0.016,
      "num_tokens": 158194.0,
      "reward": 0.3384375274181366,
      "reward_std": 0.16842570900917053,
      "rewards/rollout_reward_func/mean": 0.3384375274181366,
      "rewards/rollout_reward_func/std": 0.27340278029441833,
      "sampling/importance_sampling_ratio/max": 1.9602876901626587,
      "sampling/importance_sampling_ratio/mean": 0.992855966091156,
      "sampling/importance_sampling_ratio/min": 0.46628525853157043,
      "sampling/sampling_logp_difference/max": 0.6929764747619629,
      "sampling/sampling_logp_difference/mean": 0.04201715067028999,
      "step": 2,
      "step_time": 13.260429134000105
    },
    {
      "clip_ratio/high_max": 0.04315628902986646,
      "clip_ratio/high_mean": 0.012242560740560293,
      "clip_ratio/low_mean": 0.011964043835178018,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.024206604342907667,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2776.0,
      "completions/max_terminated_length": 2776.0,
      "completions/mean_length": 1875.09375,
      "completions/mean_terminated_length": 1875.09375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.38934508711099625,
      "epoch": 0.00024,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 2.2311601638793945,
      "kl": 0.003617420152295381,
      "learning_rate": 3.4285714285714286e-07,
      "loss": -0.0954,
      "num_tokens": 230320.0,
      "reward": 0.4612500071525574,
      "reward_std": 0.22380851209163666,
      "rewards/rollout_reward_func/mean": 0.4612500071525574,
      "rewards/rollout_reward_func/std": 0.3984546363353729,
      "sampling/importance_sampling_ratio/max": 1.6067352294921875,
      "sampling/importance_sampling_ratio/mean": 0.9242645502090454,
      "sampling/importance_sampling_ratio/min": 0.17279618978500366,
      "sampling/sampling_logp_difference/max": 1.4119317531585693,
      "sampling/sampling_logp_difference/mean": 0.045969706028699875,
      "step": 3,
      "step_time": 12.304947445999915
    },
    {
      "clip_ratio/high_max": 0.023281023371964693,
      "clip_ratio/high_mean": 0.012716594734229147,
      "clip_ratio/low_mean": 0.01039634458720684,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.023112939670681953,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2783.0,
      "completions/max_terminated_length": 2783.0,
      "completions/mean_length": 2251.84375,
      "completions/mean_terminated_length": 2251.84375,
      "completions/min_length": 1562.0,
      "completions/min_terminated_length": 1562.0,
      "entropy": 0.4188930094242096,
      "epoch": 0.00032,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 2.3432626724243164,
      "kl": 0.005323103512637317,
      "learning_rate": 5.142857142857143e-07,
      "loss": -0.1037,
      "num_tokens": 315875.0,
      "reward": 0.2640625238418579,
      "reward_std": 0.07438889145851135,
      "rewards/rollout_reward_func/mean": 0.2640625238418579,
      "rewards/rollout_reward_func/std": 0.09810657054185867,
      "sampling/importance_sampling_ratio/max": 2.92923903465271,
      "sampling/importance_sampling_ratio/mean": 1.0071074962615967,
      "sampling/importance_sampling_ratio/min": 0.30356213450431824,
      "sampling/sampling_logp_difference/max": 0.9253432750701904,
      "sampling/sampling_logp_difference/mean": 0.04933081567287445,
      "step": 4,
      "step_time": 13.287707804000092
    },
    {
      "clip_ratio/high_max": 0.04239537985995412,
      "clip_ratio/high_mean": 0.018673060229048133,
      "clip_ratio/low_mean": 0.0042297979816794395,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022902858443558216,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2790.0,
      "completions/max_terminated_length": 2790.0,
      "completions/mean_length": 2197.3125,
      "completions/mean_terminated_length": 2197.3125,
      "completions/min_length": 1559.0,
      "completions/min_terminated_length": 1559.0,
      "entropy": 0.4414307102560997,
      "epoch": 0.0004,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 2.470518112182617,
      "kl": 0.004553150560241193,
      "learning_rate": 6.857142857142857e-07,
      "loss": 0.1372,
      "num_tokens": 399370.0,
      "reward": 0.40281248092651367,
      "reward_std": 0.16662904620170593,
      "rewards/rollout_reward_func/mean": 0.40281248092651367,
      "rewards/rollout_reward_func/std": 0.3357921242713928,
      "sampling/importance_sampling_ratio/max": 2.2576870918273926,
      "sampling/importance_sampling_ratio/mean": 1.0002690553665161,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.462371826171875,
      "sampling/sampling_logp_difference/mean": 0.053694289177656174,
      "step": 5,
      "step_time": 13.068715858000132
    },
    {
      "clip_ratio/high_max": 0.02923969691619277,
      "clip_ratio/high_mean": 0.01021690119523555,
      "clip_ratio/low_mean": 0.01101089478470385,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.021227796096354723,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2428.0,
      "completions/max_terminated_length": 2428.0,
      "completions/mean_length": 1826.09375,
      "completions/mean_terminated_length": 1826.09375,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.37763065844774246,
      "epoch": 0.00048,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.4737789630889893,
      "kl": 0.003610707528423518,
      "learning_rate": 8.571428571428571e-07,
      "loss": 0.0212,
      "num_tokens": 469858.0,
      "reward": 0.4584375023841858,
      "reward_std": 0.2892817258834839,
      "rewards/rollout_reward_func/mean": 0.4584375023841858,
      "rewards/rollout_reward_func/std": 0.4035496413707733,
      "sampling/importance_sampling_ratio/max": 1.8672934770584106,
      "sampling/importance_sampling_ratio/mean": 0.9250987768173218,
      "sampling/importance_sampling_ratio/min": 0.2111542820930481,
      "sampling/sampling_logp_difference/max": 1.105020523071289,
      "sampling/sampling_logp_difference/mean": 0.04392547905445099,
      "step": 6,
      "step_time": 11.808095918000163
    },
    {
      "clip_ratio/high_max": 0.02163859363645315,
      "clip_ratio/high_mean": 0.007195362821221352,
      "clip_ratio/low_mean": 0.009288194705732167,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01648355764336884,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2799.0,
      "completions/max_terminated_length": 2799.0,
      "completions/mean_length": 2101.9375,
      "completions/mean_terminated_length": 2101.9375,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.4031049609184265,
      "epoch": 0.00056,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 2.241011142730713,
      "kl": 0.004900285159237683,
      "learning_rate": 1.0285714285714286e-06,
      "loss": 0.0307,
      "num_tokens": 549695.0,
      "reward": 0.32218751311302185,
      "reward_std": 0.10592572391033173,
      "rewards/rollout_reward_func/mean": 0.32218751311302185,
      "rewards/rollout_reward_func/std": 0.22224271297454834,
      "sampling/importance_sampling_ratio/max": 2.7520875930786133,
      "sampling/importance_sampling_ratio/mean": 0.9687752723693848,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.8389774560928345,
      "sampling/sampling_logp_difference/mean": 0.043909620493650436,
      "step": 7,
      "step_time": 13.085814365000147
    },
    {
      "clip_ratio/high_max": 0.029183519072830677,
      "clip_ratio/high_mean": 0.008625667076557875,
      "clip_ratio/low_mean": 0.016130636679008603,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02475630398839712,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2794.0,
      "completions/max_terminated_length": 2794.0,
      "completions/mean_length": 2250.71875,
      "completions/mean_terminated_length": 2250.71875,
      "completions/min_length": 1570.0,
      "completions/min_terminated_length": 1570.0,
      "entropy": 0.43513813614845276,
      "epoch": 0.00064,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 2.6571757793426514,
      "kl": 0.0038885354879312217,
      "learning_rate": 1.2000000000000002e-06,
      "loss": -0.0896,
      "num_tokens": 634822.0,
      "reward": 0.30375000834465027,
      "reward_std": 0.11063194274902344,
      "rewards/rollout_reward_func/mean": 0.30375000834465027,
      "rewards/rollout_reward_func/std": 0.22577106952667236,
      "sampling/importance_sampling_ratio/max": 2.24173641204834,
      "sampling/importance_sampling_ratio/mean": 0.9777867794036865,
      "sampling/importance_sampling_ratio/min": 0.4010058343410492,
      "sampling/sampling_logp_difference/max": 0.9179394245147705,
      "sampling/sampling_logp_difference/mean": 0.0499531514942646,
      "step": 8,
      "step_time": 13.055169309000007
    },
    {
      "clip_ratio/high_max": 0.014742525294423103,
      "clip_ratio/high_mean": 0.003685631323605776,
      "clip_ratio/low_mean": 0.008176195668056607,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011861827224493027,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2810.0,
      "completions/max_terminated_length": 2810.0,
      "completions/mean_length": 1657.03125,
      "completions/mean_terminated_length": 1657.03125,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.39970337599515915,
      "epoch": 0.00072,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.901354432106018,
      "kl": 0.005712996702641249,
      "learning_rate": 1.3714285714285715e-06,
      "loss": 0.0004,
      "num_tokens": 699616.0,
      "reward": 0.3528124690055847,
      "reward_std": 0.20240315794944763,
      "rewards/rollout_reward_func/mean": 0.3528124690055847,
      "rewards/rollout_reward_func/std": 0.3597510755062103,
      "sampling/importance_sampling_ratio/max": 2.387613296508789,
      "sampling/importance_sampling_ratio/mean": 1.0771517753601074,
      "sampling/importance_sampling_ratio/min": 0.5435174703598022,
      "sampling/sampling_logp_difference/max": 0.6833771467208862,
      "sampling/sampling_logp_difference/mean": 0.04181923717260361,
      "step": 9,
      "step_time": 13.294292020000057
    },
    {
      "clip_ratio/high_max": 0.03906210558488965,
      "clip_ratio/high_mean": 0.015391179244033992,
      "clip_ratio/low_mean": 0.0073633925057947636,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.022754571866244078,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2793.0,
      "completions/max_terminated_length": 2793.0,
      "completions/mean_length": 2214.53125,
      "completions/mean_terminated_length": 2214.53125,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.41637370735406876,
      "epoch": 0.0008,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 2.541189670562744,
      "kl": 0.004748326842673123,
      "learning_rate": 1.5428571428571428e-06,
      "loss": -0.0168,
      "num_tokens": 783707.0,
      "reward": 0.4284375309944153,
      "reward_std": 0.1260128915309906,
      "rewards/rollout_reward_func/mean": 0.4284375309944153,
      "rewards/rollout_reward_func/std": 0.3622608780860901,
      "sampling/importance_sampling_ratio/max": 2.2261769771575928,
      "sampling/importance_sampling_ratio/mean": 1.042180061340332,
      "sampling/importance_sampling_ratio/min": 0.2320551723241806,
      "sampling/sampling_logp_difference/max": 1.021528959274292,
      "sampling/sampling_logp_difference/mean": 0.04730905592441559,
      "step": 10,
      "step_time": 13.637110585000073
    },
    {
      "clip_ratio/high_max": 0.010990338400006294,
      "clip_ratio/high_mean": 0.0027475846000015736,
      "clip_ratio/low_mean": 0.0016025641234591603,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004350148723460734,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2793.0,
      "completions/max_terminated_length": 2793.0,
      "completions/mean_length": 1966.4375,
      "completions/mean_terminated_length": 1966.4375,
      "completions/min_length": 1055.0,
      "completions/min_terminated_length": 1055.0,
      "entropy": 0.4171219617128372,
      "epoch": 0.00088,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.4207873344421387,
      "kl": 0.0035471616429276764,
      "learning_rate": 1.7142857142857143e-06,
      "loss": 0.0422,
      "num_tokens": 858994.0,
      "reward": 0.4596875309944153,
      "reward_std": 0.14279377460479736,
      "rewards/rollout_reward_func/mean": 0.4596875309944153,
      "rewards/rollout_reward_func/std": 0.3725802004337311,
      "sampling/importance_sampling_ratio/max": 1.874053716659546,
      "sampling/importance_sampling_ratio/mean": 0.9027889966964722,
      "sampling/importance_sampling_ratio/min": 0.45684853196144104,
      "sampling/sampling_logp_difference/max": 0.5253086090087891,
      "sampling/sampling_logp_difference/mean": 0.0447448305785656,
      "step": 11,
      "step_time": 12.90678849699998
    },
    {
      "clip_ratio/high_max": 0.039288708940148354,
      "clip_ratio/high_mean": 0.017087680520489812,
      "clip_ratio/low_mean": 0.008439590455964208,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02552727097645402,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2782.0,
      "completions/max_terminated_length": 2782.0,
      "completions/mean_length": 1859.5625,
      "completions/mean_terminated_length": 1859.5625,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.42384539544582367,
      "epoch": 0.00096,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 2.1793887615203857,
      "kl": 0.004719441349152476,
      "learning_rate": 1.8857142857142858e-06,
      "loss": -0.0501,
      "num_tokens": 930647.0,
      "reward": 0.5653125047683716,
      "reward_std": 0.09132834523916245,
      "rewards/rollout_reward_func/mean": 0.5653125047683716,
      "rewards/rollout_reward_func/std": 0.4122869372367859,
      "sampling/importance_sampling_ratio/max": 1.968488097190857,
      "sampling/importance_sampling_ratio/mean": 1.1238960027694702,
      "sampling/importance_sampling_ratio/min": 0.5891481637954712,
      "sampling/sampling_logp_difference/max": 0.9189000129699707,
      "sampling/sampling_logp_difference/mean": 0.045362215489149094,
      "step": 12,
      "step_time": 12.105040577999944
    },
    {
      "clip_ratio/high_max": 0.03163956617936492,
      "clip_ratio/high_mean": 0.009185401839204133,
      "clip_ratio/low_mean": 0.01267810445278883,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.021863506408408284,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2794.0,
      "completions/max_terminated_length": 2794.0,
      "completions/mean_length": 1744.0625,
      "completions/mean_terminated_length": 1744.0625,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.38077671080827713,
      "epoch": 0.00104,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.601933240890503,
      "kl": 0.0043588640401139855,
      "learning_rate": 2.0571428571428573e-06,
      "loss": -0.0271,
      "num_tokens": 998268.0,
      "reward": 0.5040624737739563,
      "reward_std": 0.3648141622543335,
      "rewards/rollout_reward_func/mean": 0.5040624737739563,
      "rewards/rollout_reward_func/std": 0.4420104920864105,
      "sampling/importance_sampling_ratio/max": 2.2825241088867188,
      "sampling/importance_sampling_ratio/mean": 1.0028969049453735,
      "sampling/importance_sampling_ratio/min": 0.37051475048065186,
      "sampling/sampling_logp_difference/max": 0.6929263472557068,
      "sampling/sampling_logp_difference/mean": 0.043037254363298416,
      "step": 13,
      "step_time": 12.434848872999964
    },
    {
      "clip_ratio/high_max": 0.0682385629042983,
      "clip_ratio/high_mean": 0.022985405288636684,
      "clip_ratio/low_mean": 0.0055555556900799274,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.028540961910039186,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2797.0,
      "completions/max_terminated_length": 2797.0,
      "completions/mean_length": 2003.40625,
      "completions/mean_terminated_length": 2003.40625,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.40549617260694504,
      "epoch": 0.00112,
      "frac_reward_zero_std": 0.125,
      "grad_norm": 2.6730706691741943,
      "kl": 0.004465080099180341,
      "learning_rate": 2.2285714285714286e-06,
      "loss": 0.0367,
      "num_tokens": 1075200.0,
      "reward": 0.3971875011920929,
      "reward_std": 0.24656714498996735,
      "rewards/rollout_reward_func/mean": 0.3971875011920929,
      "rewards/rollout_reward_func/std": 0.3921506702899933,
      "sampling/importance_sampling_ratio/max": 2.08994197845459,
      "sampling/importance_sampling_ratio/mean": 0.9472236037254333,
      "sampling/importance_sampling_ratio/min": 0.2920815646648407,
      "sampling/sampling_logp_difference/max": 0.5747750997543335,
      "sampling/sampling_logp_difference/mean": 0.0427585169672966,
      "step": 14,
      "step_time": 13.12732943400033
    },
    {
      "clip_ratio/high_max": 0.04484127042815089,
      "clip_ratio/high_mean": 0.01285505446139723,
      "clip_ratio/low_mean": 0.008134920848533511,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.020989975426346064,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2801.0,
      "completions/max_terminated_length": 2801.0,
      "completions/mean_length": 1879.96875,
      "completions/mean_terminated_length": 1879.96875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.3704817444086075,
      "epoch": 0.0012,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.3741340637207031,
      "kl": 0.004930144699756056,
      "learning_rate": 2.4000000000000003e-06,
      "loss": -0.036,
      "num_tokens": 1147654.0,
      "reward": 0.5634374618530273,
      "reward_std": 0.2032102644443512,
      "rewards/rollout_reward_func/mean": 0.5634374618530273,
      "rewards/rollout_reward_func/std": 0.4479026794433594,
      "sampling/importance_sampling_ratio/max": 1.419919490814209,
      "sampling/importance_sampling_ratio/mean": 0.8213506937026978,
      "sampling/importance_sampling_ratio/min": 0.2297196239233017,
      "sampling/sampling_logp_difference/max": 0.9635820388793945,
      "sampling/sampling_logp_difference/mean": 0.04450097680091858,
      "step": 15,
      "step_time": 12.787183091000088
    },
    {
      "clip_ratio/high_max": 0.040403091348707676,
      "clip_ratio/high_mean": 0.019732415094040334,
      "clip_ratio/low_mean": 0.011093285749666393,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.030825700610876083,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2812.0,
      "completions/max_terminated_length": 2812.0,
      "completions/mean_length": 2226.75,
      "completions/mean_terminated_length": 2226.75,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.4390428438782692,
      "epoch": 0.00128,
      "frac_reward_zero_std": 0.125,
      "grad_norm": 3.3430113792419434,
      "kl": 0.005434123100712895,
      "learning_rate": 2.571428571428571e-06,
      "loss": 0.0608,
      "num_tokens": 1232479.0,
      "reward": 0.35874998569488525,
      "reward_std": 0.16885429620742798,
      "rewards/rollout_reward_func/mean": 0.35874998569488525,
      "rewards/rollout_reward_func/std": 0.31368517875671387,
      "sampling/importance_sampling_ratio/max": 2.1880178451538086,
      "sampling/importance_sampling_ratio/mean": 0.9618589878082275,
      "sampling/importance_sampling_ratio/min": 0.12961336970329285,
      "sampling/sampling_logp_difference/max": 0.941362738609314,
      "sampling/sampling_logp_difference/mean": 0.05188923329114914,
      "step": 16,
      "step_time": 13.173405885999728
    },
    {
      "clip_ratio/high_max": 0.04390919208526611,
      "clip_ratio/high_mean": 0.017439239425584674,
      "clip_ratio/low_mean": 0.007801226573064923,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.025240465998649597,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2434.0,
      "completions/max_terminated_length": 2434.0,
      "completions/mean_length": 1750.0625,
      "completions/mean_terminated_length": 1750.0625,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.3980557546019554,
      "epoch": 0.00136,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.602077007293701,
      "kl": 0.004334585275501013,
      "learning_rate": 2.742857142857143e-06,
      "loss": -0.0264,
      "num_tokens": 1300635.0,
      "reward": 0.38499999046325684,
      "reward_std": 0.221183180809021,
      "rewards/rollout_reward_func/mean": 0.38499999046325684,
      "rewards/rollout_reward_func/std": 0.34839722514152527,
      "sampling/importance_sampling_ratio/max": 1.7235372066497803,
      "sampling/importance_sampling_ratio/mean": 0.9467421770095825,
      "sampling/importance_sampling_ratio/min": 0.2654297649860382,
      "sampling/sampling_logp_difference/max": 0.7773740887641907,
      "sampling/sampling_logp_difference/mean": 0.04712219163775444,
      "step": 17,
      "step_time": 11.411276259999795
    },
    {
      "clip_ratio/high_max": 0.04594441968947649,
      "clip_ratio/high_mean": 0.013718247646465898,
      "clip_ratio/low_mean": 0.004949534311890602,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018667781492695212,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2798.0,
      "completions/max_terminated_length": 2798.0,
      "completions/mean_length": 2309.09375,
      "completions/mean_terminated_length": 2309.09375,
      "completions/min_length": 1055.0,
      "completions/min_terminated_length": 1055.0,
      "entropy": 0.4307108670473099,
      "epoch": 0.00144,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.9505234956741333,
      "kl": 0.004669323505368084,
      "learning_rate": 2.9142857142857142e-06,
      "loss": 0.0981,
      "num_tokens": 1388529.0,
      "reward": 0.3696874976158142,
      "reward_std": 0.155008003115654,
      "rewards/rollout_reward_func/mean": 0.3696874976158142,
      "rewards/rollout_reward_func/std": 0.28414538502693176,
      "sampling/importance_sampling_ratio/max": 1.8336728811264038,
      "sampling/importance_sampling_ratio/mean": 0.9352109432220459,
      "sampling/importance_sampling_ratio/min": 0.28059616684913635,
      "sampling/sampling_logp_difference/max": 1.0694303512573242,
      "sampling/sampling_logp_difference/mean": 0.05270082503557205,
      "step": 18,
      "step_time": 13.463537552000162
    },
    {
      "clip_ratio/high_max": 0.03351574344560504,
      "clip_ratio/high_mean": 0.017963151913136244,
      "clip_ratio/low_mean": 0.005672972998581827,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.023636124562472105,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2782.0,
      "completions/max_terminated_length": 2782.0,
      "completions/mean_length": 2203.90625,
      "completions/mean_terminated_length": 2203.90625,
      "completions/min_length": 1564.0,
      "completions/min_terminated_length": 1564.0,
      "entropy": 0.4163732975721359,
      "epoch": 0.00152,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 2.54580020904541,
      "kl": 0.0039027896127663553,
      "learning_rate": 3.0857142857142855e-06,
      "loss": -0.0385,
      "num_tokens": 1472480.0,
      "reward": 0.2887499928474426,
      "reward_std": 0.1067335307598114,
      "rewards/rollout_reward_func/mean": 0.2887499928474426,
      "rewards/rollout_reward_func/std": 0.17496080696582794,
      "sampling/importance_sampling_ratio/max": 2.46917724609375,
      "sampling/importance_sampling_ratio/mean": 1.0520013570785522,
      "sampling/importance_sampling_ratio/min": 0.31319668889045715,
      "sampling/sampling_logp_difference/max": 0.6751515865325928,
      "sampling/sampling_logp_difference/mean": 0.04795370250940323,
      "step": 19,
      "step_time": 12.881922476999762
    },
    {
      "clip_ratio/high_max": 0.03289473615586758,
      "clip_ratio/high_mean": 0.016854635905474424,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016854635905474424,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2803.0,
      "completions/max_terminated_length": 2803.0,
      "completions/mean_length": 1839.5625,
      "completions/mean_terminated_length": 1839.5625,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.3896697387099266,
      "epoch": 0.0016,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 3.0274949073791504,
      "kl": 0.004332752665504813,
      "learning_rate": 3.257142857142857e-06,
      "loss": 0.1087,
      "num_tokens": 1543546.0,
      "reward": 0.4571874737739563,
      "reward_std": 0.20620864629745483,
      "rewards/rollout_reward_func/mean": 0.4571874737739563,
      "rewards/rollout_reward_func/std": 0.38446637988090515,
      "sampling/importance_sampling_ratio/max": 2.2497854232788086,
      "sampling/importance_sampling_ratio/mean": 0.9864073395729065,
      "sampling/importance_sampling_ratio/min": 0.3370327055454254,
      "sampling/sampling_logp_difference/max": 0.9195313453674316,
      "sampling/sampling_logp_difference/mean": 0.04598519578576088,
      "step": 20,
      "step_time": 13.28698261799991
    },
    {
      "clip_ratio/high_max": 0.047807968221604824,
      "clip_ratio/high_mean": 0.017668311716988683,
      "clip_ratio/low_mean": 0.0087070451118052,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.026375357527285814,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2784.0,
      "completions/max_terminated_length": 2784.0,
      "completions/mean_length": 1916.84375,
      "completions/mean_terminated_length": 1916.84375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.3857065215706825,
      "epoch": 0.00168,
      "frac_reward_zero_std": 0.125,
      "grad_norm": 2.2928388118743896,
      "kl": 0.0030007859459146857,
      "learning_rate": 3.4285714285714285e-06,
      "loss": 0.0691,
      "num_tokens": 1617299.0,
      "reward": 0.5199999809265137,
      "reward_std": 0.24810142815113068,
      "rewards/rollout_reward_func/mean": 0.5199999809265137,
      "rewards/rollout_reward_func/std": 0.44394853711128235,
      "sampling/importance_sampling_ratio/max": 1.9692103862762451,
      "sampling/importance_sampling_ratio/mean": 1.0206944942474365,
      "sampling/importance_sampling_ratio/min": 0.37676262855529785,
      "sampling/sampling_logp_difference/max": 0.5263509750366211,
      "sampling/sampling_logp_difference/mean": 0.04122690111398697,
      "step": 21,
      "step_time": 12.188486421000334
    },
    {
      "clip_ratio/high_max": 0.040458154398947954,
      "clip_ratio/high_mean": 0.011364538804627955,
      "clip_ratio/low_mean": 0.006526540499180555,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017891079653054476,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2801.0,
      "completions/max_terminated_length": 2801.0,
      "completions/mean_length": 1878.4375,
      "completions/mean_terminated_length": 1878.4375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.3931129276752472,
      "epoch": 0.00176,
      "frac_reward_zero_std": 0.125,
      "grad_norm": 1.7913806438446045,
      "kl": 0.004387594643048942,
      "learning_rate": 3.6e-06,
      "loss": -0.0657,
      "num_tokens": 1690165.0,
      "reward": 0.48281246423721313,
      "reward_std": 0.24489575624465942,
      "rewards/rollout_reward_func/mean": 0.48281246423721313,
      "rewards/rollout_reward_func/std": 0.42833685874938965,
      "sampling/importance_sampling_ratio/max": 2.001044750213623,
      "sampling/importance_sampling_ratio/mean": 0.8716533780097961,
      "sampling/importance_sampling_ratio/min": 0.21946659684181213,
      "sampling/sampling_logp_difference/max": 0.6549723148345947,
      "sampling/sampling_logp_difference/mean": 0.04290828853845596,
      "step": 22,
      "step_time": 12.900562208999872
    },
    {
      "clip_ratio/high_max": 0.012908496893942356,
      "clip_ratio/high_mean": 0.003227124223485589,
      "clip_ratio/low_mean": 0.0013888889225199819,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004616013146005571,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2795.0,
      "completions/max_terminated_length": 2795.0,
      "completions/mean_length": 1921.96875,
      "completions/mean_terminated_length": 1921.96875,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.3991682603955269,
      "epoch": 0.00184,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 2.757072687149048,
      "kl": 0.002871026110369712,
      "learning_rate": 3.7714285714285716e-06,
      "loss": -0.0322,
      "num_tokens": 1764351.0,
      "reward": 0.4725000262260437,
      "reward_std": 0.15098075568675995,
      "rewards/rollout_reward_func/mean": 0.4725000262260437,
      "rewards/rollout_reward_func/std": 0.3937331438064575,
      "sampling/importance_sampling_ratio/max": 2.473691463470459,
      "sampling/importance_sampling_ratio/mean": 1.0277502536773682,
      "sampling/importance_sampling_ratio/min": 0.3683130145072937,
      "sampling/sampling_logp_difference/max": 0.8325839042663574,
      "sampling/sampling_logp_difference/mean": 0.041013769805431366,
      "step": 23,
      "step_time": 12.923486550999996
    },
    {
      "clip_ratio/high_max": 0.022044573910534382,
      "clip_ratio/high_mean": 0.0072472544852644205,
      "clip_ratio/low_mean": 0.007787698996253312,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015034952783025801,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2791.0,
      "completions/max_terminated_length": 2791.0,
      "completions/mean_length": 2014.1875,
      "completions/mean_terminated_length": 2014.1875,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.39251676946878433,
      "epoch": 0.00192,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.9654884338378906,
      "kl": 0.0081728242803365,
      "learning_rate": 3.942857142857143e-06,
      "loss": -0.0383,
      "num_tokens": 1841628.0,
      "reward": 0.35874998569488525,
      "reward_std": 0.21719886362552643,
      "rewards/rollout_reward_func/mean": 0.35874998569488525,
      "rewards/rollout_reward_func/std": 0.31252095103263855,
      "sampling/importance_sampling_ratio/max": 2.0834484100341797,
      "sampling/importance_sampling_ratio/mean": 0.9893499612808228,
      "sampling/importance_sampling_ratio/min": 0.06596492230892181,
      "sampling/sampling_logp_difference/max": 1.764291524887085,
      "sampling/sampling_logp_difference/mean": 0.05037356913089752,
      "step": 24,
      "step_time": 12.55188547500029
    },
    {
      "clip_ratio/high_max": 0.03194444486871362,
      "clip_ratio/high_mean": 0.009474206599406898,
      "clip_ratio/low_mean": 0.004620927385985851,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.014095134101808071,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2432.0,
      "completions/max_terminated_length": 2432.0,
      "completions/mean_length": 1997.46875,
      "completions/mean_terminated_length": 1997.46875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.40253835916519165,
      "epoch": 0.002,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 2.1649582386016846,
      "kl": 0.007934511464554816,
      "learning_rate": 4.114285714285715e-06,
      "loss": -0.084,
      "num_tokens": 1918276.0,
      "reward": 0.3425000011920929,
      "reward_std": 0.16030071675777435,
      "rewards/rollout_reward_func/mean": 0.3425000011920929,
      "rewards/rollout_reward_func/std": 0.27845191955566406,
      "sampling/importance_sampling_ratio/max": 1.7379083633422852,
      "sampling/importance_sampling_ratio/mean": 1.0123233795166016,
      "sampling/importance_sampling_ratio/min": 0.21978217363357544,
      "sampling/sampling_logp_difference/max": 0.9820888042449951,
      "sampling/sampling_logp_difference/mean": 0.043975915759801865,
      "step": 25,
      "step_time": 11.6416912709999
    },
    {
      "clip_ratio/high_max": 0.057189542800188065,
      "clip_ratio/high_mean": 0.02329625654965639,
      "clip_ratio/low_mean": 0.008795286994427443,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03209154261276126,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2812.0,
      "completions/max_terminated_length": 2812.0,
      "completions/mean_length": 2010.0625,
      "completions/mean_terminated_length": 2010.0625,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.3653796315193176,
      "epoch": 0.00208,
      "frac_reward_zero_std": 0.125,
      "grad_norm": 2.1230344772338867,
      "kl": 0.006589570315554738,
      "learning_rate": 4.285714285714286e-06,
      "loss": -0.0197,
      "num_tokens": 1995372.0,
      "reward": 0.4256249964237213,
      "reward_std": 0.23703671991825104,
      "rewards/rollout_reward_func/mean": 0.4256249964237213,
      "rewards/rollout_reward_func/std": 0.3602412939071655,
      "sampling/importance_sampling_ratio/max": 1.7632914781570435,
      "sampling/importance_sampling_ratio/mean": 0.9213794469833374,
      "sampling/importance_sampling_ratio/min": 0.4378761649131775,
      "sampling/sampling_logp_difference/max": 0.56688392162323,
      "sampling/sampling_logp_difference/mean": 0.03944293037056923,
      "step": 26,
      "step_time": 13.15109654299954
    },
    {
      "clip_ratio/high_max": 0.04949874710291624,
      "clip_ratio/high_mean": 0.02149919094517827,
      "clip_ratio/low_mean": 0.00766741088591516,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02916660183109343,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2785.0,
      "completions/max_terminated_length": 2785.0,
      "completions/mean_length": 1842.0625,
      "completions/mean_terminated_length": 1842.0625,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.4269302785396576,
      "epoch": 0.00216,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 1.6013935804367065,
      "kl": 0.00617267657071352,
      "learning_rate": 4.457142857142857e-06,
      "loss": -0.0345,
      "num_tokens": 2066465.0,
      "reward": 0.5221875309944153,
      "reward_std": 0.22779378294944763,
      "rewards/rollout_reward_func/mean": 0.5221875309944153,
      "rewards/rollout_reward_func/std": 0.4334239661693573,
      "sampling/importance_sampling_ratio/max": 2.312187433242798,
      "sampling/importance_sampling_ratio/mean": 0.8621585369110107,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9948511123657227,
      "sampling/sampling_logp_difference/mean": 0.051924653351306915,
      "step": 27,
      "step_time": 12.681567872999949
    },
    {
      "clip_ratio/high_max": 0.04371212236583233,
      "clip_ratio/high_mean": 0.0183574166148901,
      "clip_ratio/low_mean": 0.005908275721594691,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.024265691870823503,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2796.0,
      "completions/max_terminated_length": 2796.0,
      "completions/mean_length": 2155.5,
      "completions/mean_terminated_length": 2155.5,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.41429970413446426,
      "epoch": 0.00224,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 2.647275447845459,
      "kl": 0.010079714236781001,
      "learning_rate": 4.628571428571429e-06,
      "loss": -0.0864,
      "num_tokens": 2148817.0,
      "reward": 0.3021875023841858,
      "reward_std": 0.11279378086328506,
      "rewards/rollout_reward_func/mean": 0.3021875023841858,
      "rewards/rollout_reward_func/std": 0.23064753413200378,
      "sampling/importance_sampling_ratio/max": 2.1843345165252686,
      "sampling/importance_sampling_ratio/mean": 0.9328470230102539,
      "sampling/importance_sampling_ratio/min": 0.11585874110460281,
      "sampling/sampling_logp_difference/max": 1.9821176528930664,
      "sampling/sampling_logp_difference/mean": 0.05276907980442047,
      "step": 28,
      "step_time": 12.799536062000016
    },
    {
      "clip_ratio/high_max": 0.039141415152698755,
      "clip_ratio/high_mean": 0.019034530967473984,
      "clip_ratio/low_mean": 0.005208333372138441,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02424286410678178,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2411.0,
      "completions/max_terminated_length": 2411.0,
      "completions/mean_length": 1544.21875,
      "completions/mean_terminated_length": 1544.21875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.38873114436864853,
      "epoch": 0.00232,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.288419485092163,
      "kl": 0.008441059850156307,
      "learning_rate": 4.800000000000001e-06,
      "loss": -0.0294,
      "num_tokens": 2209518.0,
      "reward": 0.5049999952316284,
      "reward_std": 0.367961049079895,
      "rewards/rollout_reward_func/mean": 0.5049999952316284,
      "rewards/rollout_reward_func/std": 0.4586867392063141,
      "sampling/importance_sampling_ratio/max": 1.7176055908203125,
      "sampling/importance_sampling_ratio/mean": 0.8919655084609985,
      "sampling/importance_sampling_ratio/min": 0.3174732029438019,
      "sampling/sampling_logp_difference/max": 1.007685899734497,
      "sampling/sampling_logp_difference/mean": 0.043198756873607635,
      "step": 29,
      "step_time": 11.569315259000177
    },
    {
      "clip_ratio/high_max": 0.03119284799322486,
      "clip_ratio/high_mean": 0.009251700364984572,
      "clip_ratio/low_mean": 0.0032051282469183207,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012456828728318214,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2776.0,
      "completions/max_terminated_length": 2776.0,
      "completions/mean_length": 1695.40625,
      "completions/mean_terminated_length": 1695.40625,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.38929111510515213,
      "epoch": 0.0024,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 1.749756932258606,
      "kl": 0.01017191493883729,
      "learning_rate": 4.9714285714285715e-06,
      "loss": 0.0146,
      "num_tokens": 2275561.0,
      "reward": 0.5309374928474426,
      "reward_std": 0.32216140627861023,
      "rewards/rollout_reward_func/mean": 0.5309374928474426,
      "rewards/rollout_reward_func/std": 0.4390852451324463,
      "sampling/importance_sampling_ratio/max": 2.9540531635284424,
      "sampling/importance_sampling_ratio/mean": 1.0208276510238647,
      "sampling/importance_sampling_ratio/min": 0.37041175365448,
      "sampling/sampling_logp_difference/max": 0.5885751247406006,
      "sampling/sampling_logp_difference/mean": 0.04683335870504379,
      "step": 30,
      "step_time": 12.191692169999897
    },
    {
      "clip_ratio/high_max": 0.05563905602321029,
      "clip_ratio/high_mean": 0.01747169380541891,
      "clip_ratio/low_mean": 0.008184524020180106,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02565621805842966,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2783.0,
      "completions/max_terminated_length": 2783.0,
      "completions/mean_length": 1801.09375,
      "completions/mean_terminated_length": 1801.09375,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.3914438411593437,
      "epoch": 0.00248,
      "frac_reward_zero_std": 0.125,
      "grad_norm": 2.8585875034332275,
      "kl": 0.015274998731911182,
      "learning_rate": 5.142857142857142e-06,
      "loss": 0.0419,
      "num_tokens": 2345322.0,
      "reward": 0.36281251907348633,
      "reward_std": 0.2801453769207001,
      "rewards/rollout_reward_func/mean": 0.36281251907348633,
      "rewards/rollout_reward_func/std": 0.342911958694458,
      "sampling/importance_sampling_ratio/max": 2.163181781768799,
      "sampling/importance_sampling_ratio/mean": 0.9487945437431335,
      "sampling/importance_sampling_ratio/min": 0.29707521200180054,
      "sampling/sampling_logp_difference/max": 0.7824678421020508,
      "sampling/sampling_logp_difference/mean": 0.0532098188996315,
      "step": 31,
      "step_time": 13.19187305000014
    },
    {
      "clip_ratio/high_max": 0.03187447274103761,
      "clip_ratio/high_mean": 0.018647319404408336,
      "clip_ratio/low_mean": 0.004727297928184271,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02337461756542325,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2435.0,
      "completions/max_terminated_length": 2435.0,
      "completions/mean_length": 1984.90625,
      "completions/mean_terminated_length": 1984.90625,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.416415698826313,
      "epoch": 0.00256,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 2.3030495643615723,
      "kl": 0.015865659108385444,
      "learning_rate": 5.314285714285714e-06,
      "loss": -0.0567,
      "num_tokens": 2421421.0,
      "reward": 0.3878124952316284,
      "reward_std": 0.23157384991645813,
      "rewards/rollout_reward_func/mean": 0.3878124952316284,
      "rewards/rollout_reward_func/std": 0.3412286341190338,
      "sampling/importance_sampling_ratio/max": 2.5926010608673096,
      "sampling/importance_sampling_ratio/mean": 0.9760158658027649,
      "sampling/importance_sampling_ratio/min": 0.2061164528131485,
      "sampling/sampling_logp_difference/max": 0.8063008785247803,
      "sampling/sampling_logp_difference/mean": 0.04909588024020195,
      "step": 32,
      "step_time": 11.466194520999807
    },
    {
      "clip_ratio/high_max": 0.019717262126505375,
      "clip_ratio/high_mean": 0.004929315531626344,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004929315531626344,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2809.0,
      "completions/max_terminated_length": 2809.0,
      "completions/mean_length": 2102.71875,
      "completions/mean_terminated_length": 2102.71875,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.42558059841394424,
      "epoch": 0.00264,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.5914040803909302,
      "kl": 0.010543531039729714,
      "learning_rate": 5.485714285714286e-06,
      "loss": 0.0448,
      "num_tokens": 2501867.0,
      "reward": 0.5221875309944153,
      "reward_std": 0.14279377460479736,
      "rewards/rollout_reward_func/mean": 0.5221875309944153,
      "rewards/rollout_reward_func/std": 0.4007873833179474,
      "sampling/importance_sampling_ratio/max": 1.5994207859039307,
      "sampling/importance_sampling_ratio/mean": 0.8397550582885742,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9267706871032715,
      "sampling/sampling_logp_difference/mean": 0.0471554696559906,
      "step": 33,
      "step_time": 12.975996798000097
    },
    {
      "clip_ratio/high_max": 0.040178571827709675,
      "clip_ratio/high_mean": 0.016144166933372617,
      "clip_ratio/low_mean": 0.005662594106979668,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.021806761040352285,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2441.0,
      "completions/max_terminated_length": 2441.0,
      "completions/mean_length": 1488.4375,
      "completions/mean_terminated_length": 1488.4375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.35695891827344894,
      "epoch": 0.00272,
      "frac_reward_zero_std": 0.125,
      "grad_norm": 1.6733559370040894,
      "kl": 0.020034206565469503,
      "learning_rate": 5.6571428571428576e-06,
      "loss": -0.0588,
      "num_tokens": 2560884.0,
      "reward": 0.5859375,
      "reward_std": 0.38607701659202576,
      "rewards/rollout_reward_func/mean": 0.5859375,
      "rewards/rollout_reward_func/std": 0.45654281973838806,
      "sampling/importance_sampling_ratio/max": 1.8220971822738647,
      "sampling/importance_sampling_ratio/mean": 0.9860107898712158,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9601047039031982,
      "sampling/sampling_logp_difference/mean": 0.052328821271657944,
      "step": 34,
      "step_time": 10.76481853400037
    },
    {
      "clip_ratio/high_max": 0.00657894741743803,
      "clip_ratio/high_mean": 0.003289473708719015,
      "clip_ratio/low_mean": 0.008878070977516472,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012167544686235487,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2801.0,
      "completions/max_terminated_length": 2801.0,
      "completions/mean_length": 1756.46875,
      "completions/mean_terminated_length": 1756.46875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.38564804941415787,
      "epoch": 0.0028,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.5950710773468018,
      "kl": 0.0196278584189713,
      "learning_rate": 5.8285714285714284e-06,
      "loss": 0.0794,
      "num_tokens": 2629098.0,
      "reward": 0.4750000238418579,
      "reward_std": 0.26933756470680237,
      "rewards/rollout_reward_func/mean": 0.4750000238418579,
      "rewards/rollout_reward_func/std": 0.40420371294021606,
      "sampling/importance_sampling_ratio/max": 2.8944315910339355,
      "sampling/importance_sampling_ratio/mean": 1.212613582611084,
      "sampling/importance_sampling_ratio/min": 0.3920697867870331,
      "sampling/sampling_logp_difference/max": 0.7614344358444214,
      "sampling/sampling_logp_difference/mean": 0.050811417400836945,
      "step": 35,
      "step_time": 12.117880444999855
    },
    {
      "clip_ratio/high_max": 0.032855731435120106,
      "clip_ratio/high_mean": 0.008213932858780026,
      "clip_ratio/low_mean": 0.008068988332524896,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016282920725643635,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2819.0,
      "completions/max_terminated_length": 2819.0,
      "completions/mean_length": 2214.375,
      "completions/mean_terminated_length": 2214.375,
      "completions/min_length": 1579.0,
      "completions/min_terminated_length": 1579.0,
      "entropy": 0.4132639244198799,
      "epoch": 0.00288,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.4248710870742798,
      "kl": 0.04949819762259722,
      "learning_rate": 6e-06,
      "loss": -0.1152,
      "num_tokens": 2713433.0,
      "reward": 0.3043749928474426,
      "reward_std": 0.08011817932128906,
      "rewards/rollout_reward_func/mean": 0.3043749928474426,
      "rewards/rollout_reward_func/std": 0.16871310770511627,
      "sampling/importance_sampling_ratio/max": 2.279515504837036,
      "sampling/importance_sampling_ratio/mean": 1.0208816528320312,
      "sampling/importance_sampling_ratio/min": 0.2197788804769516,
      "sampling/sampling_logp_difference/max": 1.5309280157089233,
      "sampling/sampling_logp_difference/mean": 0.05491582304239273,
      "step": 36,
      "step_time": 13.165009270000155
    },
    {
      "clip_ratio/high_max": 0.01785714365541935,
      "clip_ratio/high_mean": 0.004464285913854837,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006696428870782256,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2803.0,
      "completions/max_terminated_length": 2803.0,
      "completions/mean_length": 1736.1875,
      "completions/mean_terminated_length": 1736.1875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.3515569269657135,
      "epoch": 0.00296,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.0670298337936401,
      "kl": 0.025617226026952267,
      "learning_rate": 5.999999982184864e-06,
      "loss": 0.0221,
      "num_tokens": 2780777.0,
      "reward": 0.4387500286102295,
      "reward_std": 0.25966876745224,
      "rewards/rollout_reward_func/mean": 0.4387500286102295,
      "rewards/rollout_reward_func/std": 0.3832606077194214,
      "sampling/importance_sampling_ratio/max": 2.3271644115448,
      "sampling/importance_sampling_ratio/mean": 1.0649113655090332,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.0678925514221191,
      "sampling/sampling_logp_difference/mean": 0.05666026473045349,
      "step": 37,
      "step_time": 12.593250806000015
    },
    {
      "clip_ratio/high_max": 0.028383397962898016,
      "clip_ratio/high_mean": 0.010161041049286723,
      "clip_ratio/low_mean": 0.006483843666501343,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.016644884599372745,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2777.0,
      "completions/max_terminated_length": 2777.0,
      "completions/mean_length": 1819.5625,
      "completions/mean_terminated_length": 1819.5625,
      "completions/min_length": 1056.0,
      "completions/min_terminated_length": 1056.0,
      "entropy": 0.38034912198781967,
      "epoch": 0.00304,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 2.0448880195617676,
      "kl": 0.04296189732849598,
      "learning_rate": 5.999999928739459e-06,
      "loss": -0.0115,
      "num_tokens": 2851032.0,
      "reward": 0.6024999618530273,
      "reward_std": 0.2617889940738678,
      "rewards/rollout_reward_func/mean": 0.6024999618530273,
      "rewards/rollout_reward_func/std": 0.44098126888275146,
      "sampling/importance_sampling_ratio/max": 2.681164503097534,
      "sampling/importance_sampling_ratio/mean": 1.0418896675109863,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.4294462203979492,
      "sampling/sampling_logp_difference/mean": 0.0609976202249527,
      "step": 38,
      "step_time": 12.55964067500031
    },
    {
      "clip_ratio/high_max": 0.047167123295366764,
      "clip_ratio/high_mean": 0.014736625598743558,
      "clip_ratio/low_mean": 0.004429678898304701,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01916630449704826,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2820.0,
      "completions/max_terminated_length": 2820.0,
      "completions/mean_length": 2000.0,
      "completions/mean_terminated_length": 2000.0,
      "completions/min_length": 1055.0,
      "completions/min_terminated_length": 1055.0,
      "entropy": 0.4035666435956955,
      "epoch": 0.00312,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 1.904247760772705,
      "kl": 0.03608058113604784,
      "learning_rate": 5.999999839663784e-06,
      "loss": -0.1975,
      "num_tokens": 2927712.0,
      "reward": 0.3853124976158142,
      "reward_std": 0.1657649129629135,
      "rewards/rollout_reward_func/mean": 0.3853124976158142,
      "rewards/rollout_reward_func/std": 0.31012988090515137,
      "sampling/importance_sampling_ratio/max": 2.3516104221343994,
      "sampling/importance_sampling_ratio/mean": 0.8599222898483276,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.4187037944793701,
      "sampling/sampling_logp_difference/mean": 0.05978023633360863,
      "step": 39,
      "step_time": 12.440508590000036
    },
    {
      "clip_ratio/high_max": 0.04069459065794945,
      "clip_ratio/high_mean": 0.017941734986379743,
      "clip_ratio/low_mean": 0.0016447368543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01958647184073925,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2423.0,
      "completions/max_terminated_length": 2423.0,
      "completions/mean_length": 1889.0625,
      "completions/mean_terminated_length": 1889.0625,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.42887038737535477,
      "epoch": 0.0032,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 2.507852077484131,
      "kl": 0.031137569807469845,
      "learning_rate": 5.99999971495784e-06,
      "loss": -0.0375,
      "num_tokens": 3000212.0,
      "reward": 0.38593751192092896,
      "reward_std": 0.16842570900917053,
      "rewards/rollout_reward_func/mean": 0.38593751192092896,
      "rewards/rollout_reward_func/std": 0.35313212871551514,
      "sampling/importance_sampling_ratio/max": 1.8619109392166138,
      "sampling/importance_sampling_ratio/mean": 0.8876512050628662,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.8854889869689941,
      "sampling/sampling_logp_difference/mean": 0.0671561062335968,
      "step": 40,
      "step_time": 11.693177195999851
    },
    {
      "clip_ratio/high_max": 0.02651259582489729,
      "clip_ratio/high_mean": 0.006628148956224322,
      "clip_ratio/low_mean": 0.0017361111240461469,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00836426008027047,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2789.0,
      "completions/max_terminated_length": 2789.0,
      "completions/mean_length": 2136.03125,
      "completions/mean_terminated_length": 2136.03125,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.42095063626766205,
      "epoch": 0.00328,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.2850134372711182,
      "kl": 0.039208856876939535,
      "learning_rate": 5.99999955462163e-06,
      "loss": -0.0237,
      "num_tokens": 3081651.0,
      "reward": 0.3506249785423279,
      "reward_std": 0.1440507173538208,
      "rewards/rollout_reward_func/mean": 0.3506249785423279,
      "rewards/rollout_reward_func/std": 0.2683153748512268,
      "sampling/importance_sampling_ratio/max": 2.8166987895965576,
      "sampling/importance_sampling_ratio/mean": 1.0108704566955566,
      "sampling/importance_sampling_ratio/min": 0.14420194923877716,
      "sampling/sampling_logp_difference/max": 1.127936840057373,
      "sampling/sampling_logp_difference/mean": 0.06519916653633118,
      "step": 41,
      "step_time": 14.135176596000292
    },
    {
      "clip_ratio/high_max": 0.03996024373918772,
      "clip_ratio/high_mean": 0.012911256635561585,
      "clip_ratio/low_mean": 0.004817708395421505,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01772896503098309,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2767.0,
      "completions/max_terminated_length": 2767.0,
      "completions/mean_length": 1934.65625,
      "completions/mean_terminated_length": 1934.65625,
      "completions/min_length": 1055.0,
      "completions/min_terminated_length": 1055.0,
      "entropy": 0.38335342705249786,
      "epoch": 0.00336,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 2.1722676753997803,
      "kl": 0.13585597835481167,
      "learning_rate": 5.999999358655157e-06,
      "loss": -0.2418,
      "num_tokens": 3156023.0,
      "reward": 0.3475000262260437,
      "reward_std": 0.21655070781707764,
      "rewards/rollout_reward_func/mean": 0.3475000262260437,
      "rewards/rollout_reward_func/std": 0.3131937086582184,
      "sampling/importance_sampling_ratio/max": 2.6130497455596924,
      "sampling/importance_sampling_ratio/mean": 0.8806287050247192,
      "sampling/importance_sampling_ratio/min": 0.16678351163864136,
      "sampling/sampling_logp_difference/max": 2.3499860763549805,
      "sampling/sampling_logp_difference/mean": 0.06342820823192596,
      "step": 42,
      "step_time": 13.112628190999885
    },
    {
      "clip_ratio/high_max": 0.021321472711861134,
      "clip_ratio/high_mean": 0.007562511134892702,
      "clip_ratio/low_mean": 0.0038768798112869263,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011439391179010272,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2797.0,
      "completions/max_terminated_length": 2797.0,
      "completions/mean_length": 1674.3125,
      "completions/mean_terminated_length": 1674.3125,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.3885280713438988,
      "epoch": 0.00344,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.4752204418182373,
      "kl": 0.036413189955055714,
      "learning_rate": 5.999999127058423e-06,
      "loss": 0.0258,
      "num_tokens": 3221611.0,
      "reward": 0.6737500429153442,
      "reward_std": 0.25966876745224,
      "rewards/rollout_reward_func/mean": 0.6737500429153442,
      "rewards/rollout_reward_func/std": 0.4556862711906433,
      "sampling/importance_sampling_ratio/max": 2.9477226734161377,
      "sampling/importance_sampling_ratio/mean": 1.1396255493164062,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.167872667312622,
      "sampling/sampling_logp_difference/mean": 0.06658157706260681,
      "step": 43,
      "step_time": 12.020586962000152
    },
    {
      "clip_ratio/high_max": 0.036011905409395695,
      "clip_ratio/high_mean": 0.010423430823720992,
      "clip_ratio/low_mean": 0.0030159883899614215,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013439419795759022,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2814.0,
      "completions/max_terminated_length": 2814.0,
      "completions/mean_length": 2095.75,
      "completions/mean_terminated_length": 2095.75,
      "completions/min_length": 1568.0,
      "completions/min_terminated_length": 1568.0,
      "entropy": 0.39560940861701965,
      "epoch": 0.00352,
      "frac_reward_zero_std": 0.125,
      "grad_norm": 1.8694807291030884,
      "kl": 0.1402588039636612,
      "learning_rate": 5.999998859831431e-06,
      "loss": -0.1597,
      "num_tokens": 3301324.0,
      "reward": 0.40437501668930054,
      "reward_std": 0.2259407639503479,
      "rewards/rollout_reward_func/mean": 0.40437501668930054,
      "rewards/rollout_reward_func/std": 0.35422733426094055,
      "sampling/importance_sampling_ratio/max": 2.6974401473999023,
      "sampling/importance_sampling_ratio/mean": 0.8676252365112305,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.911269187927246,
      "sampling/sampling_logp_difference/mean": 0.08191373944282532,
      "step": 44,
      "step_time": 12.868387047999704
    },
    {
      "clip_ratio/high_max": 0.0369886364787817,
      "clip_ratio/high_mean": 0.011032873298972845,
      "clip_ratio/low_mean": 0.0043535883305594325,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015386461513116956,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2793.0,
      "completions/max_terminated_length": 2793.0,
      "completions/mean_length": 2412.75,
      "completions/mean_terminated_length": 2412.75,
      "completions/min_length": 1056.0,
      "completions/min_terminated_length": 1056.0,
      "entropy": 0.4342958629131317,
      "epoch": 0.0036,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.422937035560608,
      "kl": 0.11194289568811655,
      "learning_rate": 5.999998556974188e-06,
      "loss": -0.1586,
      "num_tokens": 3392626.0,
      "reward": 0.35750001668930054,
      "reward_std": 0.0949999988079071,
      "rewards/rollout_reward_func/mean": 0.35750001668930054,
      "rewards/rollout_reward_func/std": 0.260532945394516,
      "sampling/importance_sampling_ratio/max": 2.1776068210601807,
      "sampling/importance_sampling_ratio/mean": 0.852668285369873,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.2368037700653076,
      "sampling/sampling_logp_difference/mean": 0.07167594134807587,
      "step": 45,
      "step_time": 13.40532306199998
    },
    {
      "clip_ratio/high_max": 0.036038962192833424,
      "clip_ratio/high_mean": 0.012058520689606667,
      "clip_ratio/low_mean": 0.0017857142956927419,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013844234868884087,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2784.0,
      "completions/max_terminated_length": 2784.0,
      "completions/mean_length": 2018.40625,
      "completions/mean_terminated_length": 2018.40625,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.3769753500819206,
      "epoch": 0.00368,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.942874908447266,
      "kl": 0.8322499115020037,
      "learning_rate": 5.999998218486697e-06,
      "loss": -0.0692,
      "num_tokens": 3469989.0,
      "reward": 0.39250001311302185,
      "reward_std": 0.14825798571109772,
      "rewards/rollout_reward_func/mean": 0.39250001311302185,
      "rewards/rollout_reward_func/std": 0.29918164014816284,
      "sampling/importance_sampling_ratio/max": 2.446554660797119,
      "sampling/importance_sampling_ratio/mean": 0.8061342239379883,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.015519142150879,
      "sampling/sampling_logp_difference/mean": 0.07696790993213654,
      "step": 46,
      "step_time": 12.274703390000013
    },
    {
      "clip_ratio/high_max": 0.04237867519259453,
      "clip_ratio/high_mean": 0.01807057624682784,
      "clip_ratio/low_mean": 0.005178963067010045,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.023249539081007242,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2800.0,
      "completions/max_terminated_length": 2800.0,
      "completions/mean_length": 1802.75,
      "completions/mean_terminated_length": 1802.75,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.38206612318754196,
      "epoch": 0.00376,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.6050693988800049,
      "kl": 0.05531273875385523,
      "learning_rate": 5.999997844368963e-06,
      "loss": -0.0113,
      "num_tokens": 3540097.0,
      "reward": 0.4990624785423279,
      "reward_std": 0.28371256589889526,
      "rewards/rollout_reward_func/mean": 0.4990624785423279,
      "rewards/rollout_reward_func/std": 0.41065138578414917,
      "sampling/importance_sampling_ratio/max": 1.9599696397781372,
      "sampling/importance_sampling_ratio/mean": 0.8884379863739014,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.8157303333282471,
      "sampling/sampling_logp_difference/mean": 0.06130218505859375,
      "step": 47,
      "step_time": 12.33328224800016
    },
    {
      "clip_ratio/high_max": 0.0206808946095407,
      "clip_ratio/high_mean": 0.005170223652385175,
      "clip_ratio/low_mean": 0.004861111170612276,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01003133482299745,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2813.0,
      "completions/max_terminated_length": 2813.0,
      "completions/mean_length": 1976.21875,
      "completions/mean_terminated_length": 1976.21875,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.37976498901844025,
      "epoch": 0.00384,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.5906230211257935,
      "kl": 0.11688470654189587,
      "learning_rate": 5.999997434620992e-06,
      "loss": -0.1357,
      "num_tokens": 3616089.0,
      "reward": 0.437812477350235,
      "reward_std": 0.20705953240394592,
      "rewards/rollout_reward_func/mean": 0.437812477350235,
      "rewards/rollout_reward_func/std": 0.35220715403556824,
      "sampling/importance_sampling_ratio/max": 1.8663876056671143,
      "sampling/importance_sampling_ratio/mean": 0.8626433610916138,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.65832781791687,
      "sampling/sampling_logp_difference/mean": 0.06971758604049683,
      "step": 48,
      "step_time": 12.541744299999891
    },
    {
      "clip_ratio/high_max": 0.012820512987673283,
      "clip_ratio/high_mean": 0.0032051282469183207,
      "clip_ratio/low_mean": 0.0014534883666783571,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004658616613596678,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2807.0,
      "completions/max_terminated_length": 2807.0,
      "completions/mean_length": 2245.15625,
      "completions/mean_terminated_length": 2245.15625,
      "completions/min_length": 1551.0,
      "completions/min_terminated_length": 1551.0,
      "entropy": 0.4284479096531868,
      "epoch": 0.00392,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.0771631002426147,
      "kl": 0.046674114651978016,
      "learning_rate": 5.999996989242791e-06,
      "loss": -0.0014,
      "num_tokens": 3701038.0,
      "reward": 0.42624998092651367,
      "reward_std": 0.13466876745224,
      "rewards/rollout_reward_func/mean": 0.42624998092651367,
      "rewards/rollout_reward_func/std": 0.3314265012741089,
      "sampling/importance_sampling_ratio/max": 1.4823979139328003,
      "sampling/importance_sampling_ratio/mean": 0.8060042858123779,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.11665940284729,
      "sampling/sampling_logp_difference/mean": 0.0697537213563919,
      "step": 49,
      "step_time": 13.132170692999807
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2787.0,
      "completions/max_terminated_length": 2787.0,
      "completions/mean_length": 2460.71875,
      "completions/mean_terminated_length": 2460.71875,
      "completions/min_length": 2034.0,
      "completions/min_terminated_length": 2034.0,
      "entropy": 0.4269709587097168,
      "epoch": 0.004,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06742172688245773,
      "kl": 0.05730041675269604,
      "learning_rate": 5.999996508234369e-06,
      "loss": 0.0008,
      "num_tokens": 3793655.0,
      "reward": 0.30000001192092896,
      "reward_std": 0.0,
      "rewards/rollout_reward_func/mean": 0.30000001192092896,
      "rewards/rollout_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.4013469219207764,
      "sampling/importance_sampling_ratio/mean": 0.8128387928009033,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.065826416015625,
      "sampling/sampling_logp_difference/mean": 0.07448764890432358,
      "step": 50,
      "step_time": 13.017520340999681
    },
    {
      "clip_ratio/high_max": 0.03630952490493655,
      "clip_ratio/high_mean": 0.012549603707157075,
      "clip_ratio/low_mean": 0.0031565657118335366,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015706169069744647,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2785.0,
      "completions/max_terminated_length": 2785.0,
      "completions/mean_length": 1773.78125,
      "completions/mean_terminated_length": 1773.78125,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.37685880810022354,
      "epoch": 0.00408,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.299412727355957,
      "kl": 0.04002719838172197,
      "learning_rate": 5.999995991595729e-06,
      "loss": -0.0109,
      "num_tokens": 3862448.0,
      "reward": 0.5353125333786011,
      "reward_std": 0.08654377609491348,
      "rewards/rollout_reward_func/mean": 0.5353125333786011,
      "rewards/rollout_reward_func/std": 0.41608762741088867,
      "sampling/importance_sampling_ratio/max": 2.3817224502563477,
      "sampling/importance_sampling_ratio/mean": 0.9811595678329468,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.0328466892242432,
      "sampling/sampling_logp_difference/mean": 0.06913870573043823,
      "step": 51,
      "step_time": 12.60499654799969
    },
    {
      "clip_ratio/high_max": 0.03819444449618459,
      "clip_ratio/high_mean": 0.015144050237722695,
      "clip_ratio/low_mean": 0.00554396363440901,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02068801363930106,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2784.0,
      "completions/max_terminated_length": 2784.0,
      "completions/mean_length": 1786.09375,
      "completions/mean_terminated_length": 1786.09375,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.38082515448331833,
      "epoch": 0.00416,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.5871905088424683,
      "kl": 0.06744291074573994,
      "learning_rate": 5.999995439326883e-06,
      "loss": -0.0699,
      "num_tokens": 3931876.0,
      "reward": 0.6090624928474426,
      "reward_std": 0.26599711179733276,
      "rewards/rollout_reward_func/mean": 0.6090624928474426,
      "rewards/rollout_reward_func/std": 0.4591953456401825,
      "sampling/importance_sampling_ratio/max": 2.734297752380371,
      "sampling/importance_sampling_ratio/mean": 0.9665597677230835,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.065897226333618,
      "sampling/sampling_logp_difference/mean": 0.06354629993438721,
      "step": 52,
      "step_time": 13.568785395000077
    },
    {
      "clip_ratio/high_max": 0.022086466662585735,
      "clip_ratio/high_mean": 0.008820227812975645,
      "clip_ratio/low_mean": 0.0057043652050197124,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01452459313441068,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2789.0,
      "completions/max_terminated_length": 2789.0,
      "completions/mean_length": 1635.03125,
      "completions/mean_terminated_length": 1635.03125,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.36858493834733963,
      "epoch": 0.00424,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.9566222429275513,
      "kl": 0.07718627620488405,
      "learning_rate": 5.999994851427837e-06,
      "loss": 0.0822,
      "num_tokens": 3995868.0,
      "reward": 0.6918749809265137,
      "reward_std": 0.3203721046447754,
      "rewards/rollout_reward_func/mean": 0.6918749809265137,
      "rewards/rollout_reward_func/std": 0.4697249233722687,
      "sampling/importance_sampling_ratio/max": 2.7838289737701416,
      "sampling/importance_sampling_ratio/mean": 0.9136906266212463,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.7872750759124756,
      "sampling/sampling_logp_difference/mean": 0.07199069857597351,
      "step": 53,
      "step_time": 12.247058065999909
    },
    {
      "clip_ratio/high_max": 0.041652148589491844,
      "clip_ratio/high_mean": 0.013425522716715932,
      "clip_ratio/low_mean": 0.01002952002454549,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0234550426248461,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2784.0,
      "completions/max_terminated_length": 2784.0,
      "completions/mean_length": 1590.09375,
      "completions/mean_terminated_length": 1590.09375,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.37869949638843536,
      "epoch": 0.00432,
      "frac_reward_zero_std": 0.125,
      "grad_norm": 2.3980886936187744,
      "kl": 0.05773049034178257,
      "learning_rate": 5.999994227898604e-06,
      "loss": -0.0192,
      "num_tokens": 4058303.0,
      "reward": 0.4609374701976776,
      "reward_std": 0.35279375314712524,
      "rewards/rollout_reward_func/mean": 0.4609374701976776,
      "rewards/rollout_reward_func/std": 0.44058871269226074,
      "sampling/importance_sampling_ratio/max": 2.2311129570007324,
      "sampling/importance_sampling_ratio/mean": 0.9393452405929565,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9994411468505859,
      "sampling/sampling_logp_difference/mean": 0.08165294677019119,
      "step": 54,
      "step_time": 11.528886767999893
    },
    {
      "clip_ratio/high_max": 0.02447916753590107,
      "clip_ratio/high_mean": 0.0075732802506536245,
      "clip_ratio/low_mean": 0.00947712454944849,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01705040503293276,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2444.0,
      "completions/max_terminated_length": 2444.0,
      "completions/mean_length": 1789.75,
      "completions/mean_terminated_length": 1789.75,
      "completions/min_length": 1055.0,
      "completions/min_terminated_length": 1055.0,
      "entropy": 0.36519913375377655,
      "epoch": 0.0044,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 2.2972187995910645,
      "kl": 0.05506392475217581,
      "learning_rate": 5.99999356873919e-06,
      "loss": -0.1185,
      "num_tokens": 4127411.0,
      "reward": 0.40562498569488525,
      "reward_std": 0.22391541302204132,
      "rewards/rollout_reward_func/mean": 0.40562498569488525,
      "rewards/rollout_reward_func/std": 0.3422500193119049,
      "sampling/importance_sampling_ratio/max": 2.4115021228790283,
      "sampling/importance_sampling_ratio/mean": 0.9461013674736023,
      "sampling/importance_sampling_ratio/min": 0.14957794547080994,
      "sampling/sampling_logp_difference/max": 1.0122857093811035,
      "sampling/sampling_logp_difference/mean": 0.06259442120790482,
      "step": 55,
      "step_time": 11.493340414999693
    },
    {
      "clip_ratio/high_max": 0.0438775522634387,
      "clip_ratio/high_mean": 0.012457483448088169,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012457483448088169,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2799.0,
      "completions/max_terminated_length": 2799.0,
      "completions/mean_length": 2306.375,
      "completions/mean_terminated_length": 2306.375,
      "completions/min_length": 1567.0,
      "completions/min_terminated_length": 1567.0,
      "entropy": 0.40949854254722595,
      "epoch": 0.00448,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 2.023374319076538,
      "kl": 0.08688413165509701,
      "learning_rate": 5.999992873949609e-06,
      "loss": -0.0712,
      "num_tokens": 4214487.0,
      "reward": 0.296875,
      "reward_std": 0.08874999731779099,
      "rewards/rollout_reward_func/mean": 0.296875,
      "rewards/rollout_reward_func/std": 0.15228237211704254,
      "sampling/importance_sampling_ratio/max": 2.9896316528320312,
      "sampling/importance_sampling_ratio/mean": 0.968756377696991,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.835113763809204,
      "sampling/sampling_logp_difference/mean": 0.08090537041425705,
      "step": 56,
      "step_time": 13.222404719000224
    },
    {
      "clip_ratio/high_max": 0.04506416339427233,
      "clip_ratio/high_mean": 0.01424223161302507,
      "clip_ratio/low_mean": 0.002842377289198339,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01708460901863873,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2441.0,
      "completions/max_terminated_length": 2441.0,
      "completions/mean_length": 1963.0,
      "completions/mean_terminated_length": 1963.0,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.4080217182636261,
      "epoch": 0.00456,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.663516640663147,
      "kl": 0.3106076046824455,
      "learning_rate": 5.999992143529868e-06,
      "loss": -0.0796,
      "num_tokens": 4289619.0,
      "reward": 0.3934375047683716,
      "reward_std": 0.1563829779624939,
      "rewards/rollout_reward_func/mean": 0.3934375047683716,
      "rewards/rollout_reward_func/std": 0.30592650175094604,
      "sampling/importance_sampling_ratio/max": 1.4833005666732788,
      "sampling/importance_sampling_ratio/mean": 0.5795140862464905,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.2869999408721924,
      "sampling/sampling_logp_difference/mean": 0.0979442298412323,
      "step": 57,
      "step_time": 11.762371722000125
    },
    {
      "clip_ratio/high_max": 0.046875,
      "clip_ratio/high_mean": 0.01171875,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01171875,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2802.0,
      "completions/max_terminated_length": 2802.0,
      "completions/mean_length": 1879.75,
      "completions/mean_terminated_length": 1879.75,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.3892976716160774,
      "epoch": 0.00464,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.1009422540664673,
      "kl": 0.05097049381583929,
      "learning_rate": 5.999991377479982e-06,
      "loss": -0.0191,
      "num_tokens": 4362090.0,
      "reward": 0.5262500047683716,
      "reward_std": 0.1875,
      "rewards/rollout_reward_func/mean": 0.5262500047683716,
      "rewards/rollout_reward_func/std": 0.4009806215763092,
      "sampling/importance_sampling_ratio/max": 2.9964590072631836,
      "sampling/importance_sampling_ratio/mean": 1.0218505859375,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.0378296375274658,
      "sampling/sampling_logp_difference/mean": 0.06926104426383972,
      "step": 58,
      "step_time": 13.15352440300012
    },
    {
      "clip_ratio/high_max": 0.019571688026189804,
      "clip_ratio/high_mean": 0.004892922006547451,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004892922006547451,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2832.0,
      "completions/max_terminated_length": 2832.0,
      "completions/mean_length": 2202.375,
      "completions/mean_terminated_length": 2202.375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.41450754553079605,
      "epoch": 0.00472,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 2.1054370403289795,
      "kl": 0.03926007356494665,
      "learning_rate": 5.999990575799961e-06,
      "loss": 0.0595,
      "num_tokens": 4446012.0,
      "reward": 0.44343751668930054,
      "reward_std": 0.13312500715255737,
      "rewards/rollout_reward_func/mean": 0.44343751668930054,
      "rewards/rollout_reward_func/std": 0.3428213894367218,
      "sampling/importance_sampling_ratio/max": 2.236393928527832,
      "sampling/importance_sampling_ratio/mean": 0.8590089678764343,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9236248731613159,
      "sampling/sampling_logp_difference/mean": 0.06904841959476471,
      "step": 59,
      "step_time": 13.58062329900008
    },
    {
      "clip_ratio/high_max": 0.029240576550364494,
      "clip_ratio/high_mean": 0.007310144137591124,
      "clip_ratio/low_mean": 0.0030868902103975415,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010397034231573343,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2429.0,
      "completions/max_terminated_length": 2429.0,
      "completions/mean_length": 2124.9375,
      "completions/mean_terminated_length": 2124.9375,
      "completions/min_length": 1567.0,
      "completions/min_terminated_length": 1567.0,
      "entropy": 0.370839923620224,
      "epoch": 0.0048,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.5435988903045654,
      "kl": 0.08518982026726007,
      "learning_rate": 5.99998973848982e-06,
      "loss": -0.0579,
      "num_tokens": 4527051.0,
      "reward": 0.3590624928474426,
      "reward_std": 0.06796419620513916,
      "rewards/rollout_reward_func/mean": 0.3590624928474426,
      "rewards/rollout_reward_func/std": 0.22809672355651855,
      "sampling/importance_sampling_ratio/max": 2.2381787300109863,
      "sampling/importance_sampling_ratio/mean": 0.8449472188949585,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.6413207054138184,
      "sampling/sampling_logp_difference/mean": 0.069917693734169,
      "step": 60,
      "step_time": 11.73221161399988
    },
    {
      "clip_ratio/high_max": 0.02281746082007885,
      "clip_ratio/high_mean": 0.006954365293495357,
      "clip_ratio/low_mean": 0.0037499999161809683,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010704364976845682,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2767.0,
      "completions/max_terminated_length": 2767.0,
      "completions/mean_length": 1689.3125,
      "completions/mean_terminated_length": 1689.3125,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.3677019253373146,
      "epoch": 0.00488,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.9829214811325073,
      "kl": 0.058779667131602764,
      "learning_rate": 5.999988865549569e-06,
      "loss": 0.0304,
      "num_tokens": 4593095.0,
      "reward": 0.6549999713897705,
      "reward_std": 0.22813192009925842,
      "rewards/rollout_reward_func/mean": 0.6549999713897705,
      "rewards/rollout_reward_func/std": 0.45271220803260803,
      "sampling/importance_sampling_ratio/max": 1.883159875869751,
      "sampling/importance_sampling_ratio/mean": 0.8463116884231567,
      "sampling/importance_sampling_ratio/min": 0.22824469208717346,
      "sampling/sampling_logp_difference/max": 1.781625747680664,
      "sampling/sampling_logp_difference/mean": 0.06828776746988297,
      "step": 61,
      "step_time": 12.480462479000153
    },
    {
      "clip_ratio/high_max": 0.0110975606366992,
      "clip_ratio/high_mean": 0.0027743901591748,
      "clip_ratio/low_mean": 0.0014880952658131719,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004262485424987972,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2815.0,
      "completions/max_terminated_length": 2815.0,
      "completions/mean_length": 2305.15625,
      "completions/mean_terminated_length": 2305.15625,
      "completions/min_length": 1571.0,
      "completions/min_terminated_length": 1571.0,
      "entropy": 0.4012472406029701,
      "epoch": 0.00496,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.8583229780197144,
      "kl": 0.06238031107932329,
      "learning_rate": 5.999987956979225e-06,
      "loss": -0.0392,
      "num_tokens": 4680377.0,
      "reward": 0.3434374928474426,
      "reward_std": 0.08029377460479736,
      "rewards/rollout_reward_func/mean": 0.3434374928474426,
      "rewards/rollout_reward_func/std": 0.22245851159095764,
      "sampling/importance_sampling_ratio/max": 2.7665059566497803,
      "sampling/importance_sampling_ratio/mean": 0.9882571697235107,
      "sampling/importance_sampling_ratio/min": 0.059778764843940735,
      "sampling/sampling_logp_difference/max": 0.9543299674987793,
      "sampling/sampling_logp_difference/mean": 0.0674777626991272,
      "step": 62,
      "step_time": 14.56621960199982
    },
    {
      "clip_ratio/high_max": 0.05253623379394412,
      "clip_ratio/high_mean": 0.016606280929408967,
      "clip_ratio/low_mean": 0.0018382353009656072,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.018444516230374575,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2423.0,
      "completions/max_terminated_length": 2423.0,
      "completions/mean_length": 1968.9375,
      "completions/mean_terminated_length": 1968.9375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.4196172505617142,
      "epoch": 0.00504,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.905213475227356,
      "kl": 0.09828684013336897,
      "learning_rate": 5.999987012778799e-06,
      "loss": -0.0034,
      "num_tokens": 4755993.0,
      "reward": 0.33031249046325684,
      "reward_std": 0.09654378145933151,
      "rewards/rollout_reward_func/mean": 0.33031249046325684,
      "rewards/rollout_reward_func/std": 0.21877197921276093,
      "sampling/importance_sampling_ratio/max": 2.0344085693359375,
      "sampling/importance_sampling_ratio/mean": 0.7417819499969482,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.208054542541504,
      "sampling/sampling_logp_difference/mean": 0.07868118584156036,
      "step": 63,
      "step_time": 11.736785162999922
    },
    {
      "clip_ratio/high_max": 0.0055555556900799274,
      "clip_ratio/high_mean": 0.0027777778450399637,
      "clip_ratio/low_mean": 0.003794643096625805,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006572420941665769,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2814.0,
      "completions/max_terminated_length": 2814.0,
      "completions/mean_length": 1972.9375,
      "completions/mean_terminated_length": 1972.9375,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.4023704081773758,
      "epoch": 0.00512,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.3090136051177979,
      "kl": 0.12237261980772018,
      "learning_rate": 5.9999860329483104e-06,
      "loss": -0.194,
      "num_tokens": 4831827.0,
      "reward": 0.5878125429153442,
      "reward_std": 0.142506942152977,
      "rewards/rollout_reward_func/mean": 0.5878125429153442,
      "rewards/rollout_reward_func/std": 0.4488244950771332,
      "sampling/importance_sampling_ratio/max": 2.9826838970184326,
      "sampling/importance_sampling_ratio/mean": 0.9495848417282104,
      "sampling/importance_sampling_ratio/min": 0.01580546610057354,
      "sampling/sampling_logp_difference/max": 2.0490379333496094,
      "sampling/sampling_logp_difference/mean": 0.07465916872024536,
      "step": 64,
      "step_time": 12.846850890000042
    },
    {
      "clip_ratio/high_max": 0.021152781788259745,
      "clip_ratio/high_mean": 0.006812585634179413,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006812585634179413,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2800.0,
      "completions/max_terminated_length": 2800.0,
      "completions/mean_length": 2005.0,
      "completions/mean_terminated_length": 2005.0,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.40342626720666885,
      "epoch": 0.0052,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.2556148767471313,
      "kl": 0.0746797863394022,
      "learning_rate": 5.999985017487771e-06,
      "loss": -0.0305,
      "num_tokens": 4908716.0,
      "reward": 0.3818749785423279,
      "reward_std": 0.15371949970722198,
      "rewards/rollout_reward_func/mean": 0.3818749785423279,
      "rewards/rollout_reward_func/std": 0.3037022650241852,
      "sampling/importance_sampling_ratio/max": 2.2475836277008057,
      "sampling/importance_sampling_ratio/mean": 0.8594139814376831,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.4058151245117188,
      "sampling/sampling_logp_difference/mean": 0.06716296076774597,
      "step": 65,
      "step_time": 12.423915739000222
    },
    {
      "clip_ratio/high_max": 0.03018707549199462,
      "clip_ratio/high_mean": 0.007546768872998655,
      "clip_ratio/low_mean": 0.006225198740139604,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.013771967613138258,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2792.0,
      "completions/max_terminated_length": 2792.0,
      "completions/mean_length": 2048.40625,
      "completions/mean_terminated_length": 2048.40625,
      "completions/min_length": 1055.0,
      "completions/min_terminated_length": 1055.0,
      "entropy": 0.38859760761260986,
      "epoch": 0.00528,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 2.1451351642608643,
      "kl": 0.22367357090115547,
      "learning_rate": 5.999983966397197e-06,
      "loss": -0.1677,
      "num_tokens": 4987207.0,
      "reward": 0.4712499976158142,
      "reward_std": 0.18434235453605652,
      "rewards/rollout_reward_func/mean": 0.4712499976158142,
      "rewards/rollout_reward_func/std": 0.35322248935699463,
      "sampling/importance_sampling_ratio/max": 2.9422554969787598,
      "sampling/importance_sampling_ratio/mean": 0.9915428161621094,
      "sampling/importance_sampling_ratio/min": 0.016011416912078857,
      "sampling/sampling_logp_difference/max": 2.497363805770874,
      "sampling/sampling_logp_difference/mean": 0.07474374771118164,
      "step": 66,
      "step_time": 12.926716641999974
    },
    {
      "clip_ratio/high_max": 0.01376319769769907,
      "clip_ratio/high_mean": 0.005043363547883928,
      "clip_ratio/low_mean": 0.0014880952658131719,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0065314588136971,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2783.0,
      "completions/max_terminated_length": 2783.0,
      "completions/mean_length": 1772.40625,
      "completions/mean_terminated_length": 1772.40625,
      "completions/min_length": 1055.0,
      "completions/min_terminated_length": 1055.0,
      "entropy": 0.4034885838627815,
      "epoch": 0.00536,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.5488033294677734,
      "kl": 0.05104802828282118,
      "learning_rate": 5.999982879676608e-06,
      "loss": -0.041,
      "num_tokens": 5055760.0,
      "reward": 0.5737500190734863,
      "reward_std": 0.16325795650482178,
      "rewards/rollout_reward_func/mean": 0.5737500190734863,
      "rewards/rollout_reward_func/std": 0.40885162353515625,
      "sampling/importance_sampling_ratio/max": 2.278799057006836,
      "sampling/importance_sampling_ratio/mean": 0.99635910987854,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.8835396766662598,
      "sampling/sampling_logp_difference/mean": 0.06713330745697021,
      "step": 67,
      "step_time": 12.46692701400002
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.004360465100035071,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004360465100035071,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2796.0,
      "completions/max_terminated_length": 2796.0,
      "completions/mean_length": 2270.40625,
      "completions/mean_terminated_length": 2270.40625,
      "completions/min_length": 1983.0,
      "completions/min_terminated_length": 1983.0,
      "entropy": 0.4274456053972244,
      "epoch": 0.00544,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.8523308038711548,
      "kl": 0.13869191519916058,
      "learning_rate": 5.9999817573260195e-06,
      "loss": -0.1124,
      "num_tokens": 5141713.0,
      "reward": 0.2878125011920929,
      "reward_std": 0.01750694215297699,
      "rewards/rollout_reward_func/mean": 0.2878125011920929,
      "rewards/rollout_reward_func/std": 0.03849880024790764,
      "sampling/importance_sampling_ratio/max": 2.765258550643921,
      "sampling/importance_sampling_ratio/mean": 0.8237208127975464,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.7204997539520264,
      "sampling/sampling_logp_difference/mean": 0.08386299759149551,
      "step": 68,
      "step_time": 12.989918530000296
    },
    {
      "clip_ratio/high_max": 0.04849738674238324,
      "clip_ratio/high_mean": 0.016489425906911492,
      "clip_ratio/low_mean": 0.0013888889225199819,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.017878314713016152,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2441.0,
      "completions/max_terminated_length": 2441.0,
      "completions/mean_length": 1876.46875,
      "completions/mean_terminated_length": 1876.46875,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.39091238379478455,
      "epoch": 0.00552,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.4318950176239014,
      "kl": 0.09389345720410347,
      "learning_rate": 5.999980599345448e-06,
      "loss": -0.0356,
      "num_tokens": 5214177.0,
      "reward": 0.5806249976158142,
      "reward_std": 0.07874999940395355,
      "rewards/rollout_reward_func/mean": 0.5806249976158142,
      "rewards/rollout_reward_func/std": 0.4241190552711487,
      "sampling/importance_sampling_ratio/max": 1.9308501482009888,
      "sampling/importance_sampling_ratio/mean": 0.9257422089576721,
      "sampling/importance_sampling_ratio/min": 0.21397040784358978,
      "sampling/sampling_logp_difference/max": 1.7655794620513916,
      "sampling/sampling_logp_difference/mean": 0.06729073822498322,
      "step": 69,
      "step_time": 11.706464537000102
    },
    {
      "clip_ratio/high_max": 0.012202381156384945,
      "clip_ratio/high_mean": 0.0030505952890962362,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0030505952890962362,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2429.0,
      "completions/max_terminated_length": 2429.0,
      "completions/mean_length": 1901.125,
      "completions/mean_terminated_length": 1901.125,
      "completions/min_length": 1056.0,
      "completions/min_terminated_length": 1056.0,
      "entropy": 0.4174434766173363,
      "epoch": 0.0056,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.2482589483261108,
      "kl": 0.0709009887650609,
      "learning_rate": 5.999979405734914e-06,
      "loss": -0.0875,
      "num_tokens": 5287259.0,
      "reward": 0.44999998807907104,
      "reward_std": 0.19828803837299347,
      "rewards/rollout_reward_func/mean": 0.44999998807907104,
      "rewards/rollout_reward_func/std": 0.36232221126556396,
      "sampling/importance_sampling_ratio/max": 2.240993022918701,
      "sampling/importance_sampling_ratio/mean": 0.7815386652946472,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.9302306175231934,
      "sampling/sampling_logp_difference/mean": 0.07109043747186661,
      "step": 70,
      "step_time": 11.92579563199979
    },
    {
      "clip_ratio/high_max": 0.03573596617206931,
      "clip_ratio/high_mean": 0.011878836317919195,
      "clip_ratio/low_mean": 0.0017361111240461469,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01361494732555002,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2800.0,
      "completions/max_terminated_length": 2800.0,
      "completions/mean_length": 2164.34375,
      "completions/mean_terminated_length": 2164.34375,
      "completions/min_length": 1055.0,
      "completions/min_terminated_length": 1055.0,
      "entropy": 0.4176176115870476,
      "epoch": 0.00568,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.604291319847107,
      "kl": 0.0694936579093337,
      "learning_rate": 5.999978176494435e-06,
      "loss": -0.1233,
      "num_tokens": 5369772.0,
      "reward": 0.4762499928474426,
      "reward_std": 0.21075797080993652,
      "rewards/rollout_reward_func/mean": 0.4762499928474426,
      "rewards/rollout_reward_func/std": 0.3796156644821167,
      "sampling/importance_sampling_ratio/max": 2.497992992401123,
      "sampling/importance_sampling_ratio/mean": 0.8422503471374512,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.5480012893676758,
      "sampling/sampling_logp_difference/mean": 0.0728713721036911,
      "step": 71,
      "step_time": 12.894382659999792
    },
    {
      "clip_ratio/high_max": 0.02281746082007885,
      "clip_ratio/high_mean": 0.008831319864839315,
      "clip_ratio/low_mean": 0.003260501311160624,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012091821059584618,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2443.0,
      "completions/max_terminated_length": 2443.0,
      "completions/mean_length": 1719.03125,
      "completions/mean_terminated_length": 1719.03125,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.39671653509140015,
      "epoch": 0.00576,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.5779305696487427,
      "kl": 0.08434087503701448,
      "learning_rate": 5.99997691162403e-06,
      "loss": -0.0636,
      "num_tokens": 5436596.0,
      "reward": 0.6024999618530273,
      "reward_std": 0.2729267477989197,
      "rewards/rollout_reward_func/mean": 0.6024999618530273,
      "rewards/rollout_reward_func/std": 0.4505694806575775,
      "sampling/importance_sampling_ratio/max": 2.754258632659912,
      "sampling/importance_sampling_ratio/mean": 0.9120872020721436,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.2350893020629883,
      "sampling/sampling_logp_difference/mean": 0.07831829786300659,
      "step": 72,
      "step_time": 12.011820702999785
    },
    {
      "clip_ratio/high_max": 0.013888888992369175,
      "clip_ratio/high_mean": 0.0034722222480922937,
      "clip_ratio/low_mean": 0.0017361111240461469,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005208333372138441,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2780.0,
      "completions/max_terminated_length": 2780.0,
      "completions/mean_length": 1720.125,
      "completions/mean_terminated_length": 1720.125,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.35359790176153183,
      "epoch": 0.00584,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.760622262954712,
      "kl": 0.0876467265188694,
      "learning_rate": 5.99997561112372e-06,
      "loss": -0.0248,
      "num_tokens": 5503333.0,
      "reward": 0.7487499713897705,
      "reward_std": 0.2987908720970154,
      "rewards/rollout_reward_func/mean": 0.7487499713897705,
      "rewards/rollout_reward_func/std": 0.46135663986206055,
      "sampling/importance_sampling_ratio/max": 2.7877893447875977,
      "sampling/importance_sampling_ratio/mean": 0.8998199701309204,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.3734312057495117,
      "sampling/sampling_logp_difference/mean": 0.07869358360767365,
      "step": 73,
      "step_time": 13.119324159999906
    },
    {
      "clip_ratio/high_max": 0.013701201416552067,
      "clip_ratio/high_mean": 0.005070037324912846,
      "clip_ratio/low_mean": 0.0030487803742289543,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008118817582726479,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2825.0,
      "completions/max_terminated_length": 2825.0,
      "completions/mean_length": 1916.5,
      "completions/mean_terminated_length": 1916.5,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.38354693353176117,
      "epoch": 0.00592,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 0.8614935874938965,
      "kl": 0.05992862023413181,
      "learning_rate": 5.999974274993527e-06,
      "loss": 0.012,
      "num_tokens": 5576952.0,
      "reward": 0.5674999952316284,
      "reward_std": 0.20719751715660095,
      "rewards/rollout_reward_func/mean": 0.5674999952316284,
      "rewards/rollout_reward_func/std": 0.4317331612110138,
      "sampling/importance_sampling_ratio/max": 1.6715911626815796,
      "sampling/importance_sampling_ratio/mean": 0.8377959132194519,
      "sampling/importance_sampling_ratio/min": 0.196980819106102,
      "sampling/sampling_logp_difference/max": 0.913780689239502,
      "sampling/sampling_logp_difference/mean": 0.06950188428163528,
      "step": 74,
      "step_time": 12.793109332000085
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2785.0,
      "completions/max_terminated_length": 2785.0,
      "completions/mean_length": 1587.53125,
      "completions/mean_terminated_length": 1587.53125,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.35842984169721603,
      "epoch": 0.006,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.3697266578674316,
      "kl": 0.05413582641631365,
      "learning_rate": 5.99997290323347e-06,
      "loss": -0.0661,
      "num_tokens": 5639305.0,
      "reward": 0.7106249928474426,
      "reward_std": 0.30803900957107544,
      "rewards/rollout_reward_func/mean": 0.7106249928474426,
      "rewards/rollout_reward_func/std": 0.4498884081840515,
      "sampling/importance_sampling_ratio/max": 1.9579815864562988,
      "sampling/importance_sampling_ratio/mean": 0.8830677270889282,
      "sampling/importance_sampling_ratio/min": 0.15943297743797302,
      "sampling/sampling_logp_difference/max": 1.0672590732574463,
      "sampling/sampling_logp_difference/mean": 0.06444922834634781,
      "step": 75,
      "step_time": 12.032428736000156
    },
    {
      "epoch": 0.006,
      "eval_clip_ratio/high_max": 0.0,
      "eval_clip_ratio/high_mean": 0.0,
      "eval_clip_ratio/low_mean": 0.0,
      "eval_clip_ratio/low_min": 0.0,
      "eval_clip_ratio/region_mean": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 2488.6,
      "eval_completions/max_terminated_length": 2488.6,
      "eval_completions/mean_length": 1951.175,
      "eval_completions/mean_terminated_length": 1951.175,
      "eval_completions/min_length": 1361.4,
      "eval_completions/min_terminated_length": 1361.4,
      "eval_entropy": 0.37236364781856535,
      "eval_frac_reward_zero_std": 0.1,
      "eval_kl": 0.07629953697323799,
      "eval_loss": -0.0013724860036745667,
      "eval_num_tokens": 5639305.0,
      "eval_reward": 0.47524999976158144,
      "eval_reward_std": 0.37039353847503664,
      "eval_rewards/rollout_reward_func/mean": 0.47524999976158144,
      "eval_rewards/rollout_reward_func/std": 0.37039353176951406,
      "eval_runtime": 10.5117,
      "eval_samples_per_second": 0.951,
      "eval_sampling/importance_sampling_ratio/max": 1.6587244033813477,
      "eval_sampling/importance_sampling_ratio/mean": 0.8837172389030457,
      "eval_sampling/importance_sampling_ratio/min": 0.32487900257110597,
      "eval_sampling/sampling_logp_difference/max": 0.8625046908855438,
      "eval_sampling/sampling_logp_difference/mean": 0.06909476891160012,
      "eval_steps_per_second": 0.285,
      "step": 75
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 25000,
  "num_input_tokens_seen": 5639305,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}