{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.0005,
  "eval_steps": 500,
  "global_step": 50,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11287.0,
      "completions/max_terminated_length": 11287.0,
      "completions/mean_length": 8394.21875,
      "completions/mean_terminated_length": 8394.21875,
      "completions/min_length": 2887.0,
      "completions/min_terminated_length": 2887.0,
      "entropy": 0.11788358655758202,
      "epoch": 1e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6996961832046509,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": 0.1667,
      "num_tokens": 297100.0,
      "reward": -0.03610033914446831,
      "reward_std": 0.7386711835861206,
      "rewards/rollout_eval_reward_func/mean": 0.2578760087490082,
      "rewards/rollout_eval_reward_func/std": 0.30121663212776184,
      "rewards/rollout_reward_func/mean": -0.03610033914446831,
      "rewards/rollout_reward_func/std": 0.728388786315918,
      "sampling/importance_sampling_ratio/max": 1.6487237215042114,
      "sampling/importance_sampling_ratio/mean": 0.9993462562561035,
      "sampling/importance_sampling_ratio/min": 0.44016358256340027,
      "sampling/sampling_logp_difference/max": 0.8206088542938232,
      "sampling/sampling_logp_difference/mean": 0.011885393410921097,
      "step": 1,
      "step_time": 73.96733120499948
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 0.11788358655758202,
      "epoch": 2e-05,
      "grad_norm": 1.699277400970459,
      "kl": 0.0,
      "learning_rate": 2.8571428571428573e-06,
      "loss": 0.1667,
      "step": 2,
      "step_time": 33.627345983999476
    },
    {
      "clip_ratio/high_max": 0.005571161163970828,
      "clip_ratio/high_mean": 0.0031328028126154095,
      "clip_ratio/low_mean": 0.007165575778344646,
      "clip_ratio/low_min": 0.0007102272938936949,
      "clip_ratio/region_mean": 0.010298378649167717,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11163.0,
      "completions/max_terminated_length": 11163.0,
      "completions/mean_length": 8984.84375,
      "completions/mean_terminated_length": 8984.84375,
      "completions/min_length": 346.0,
      "completions/min_terminated_length": 346.0,
      "entropy": 0.14975268673151731,
      "epoch": 3e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3866604566574097,
      "kl": 0.002339825068702339,
      "learning_rate": 5.7142857142857145e-06,
      "loss": -0.2053,
      "num_tokens": 613530.0,
      "reward": -0.20636191964149475,
      "reward_std": 0.5412060022354126,
      "rewards/rollout_eval_reward_func/mean": 0.13351118564605713,
      "rewards/rollout_eval_reward_func/std": 0.24266032874584198,
      "rewards/rollout_reward_func/mean": -0.20636191964149475,
      "rewards/rollout_reward_func/std": 0.542148232460022,
      "sampling/importance_sampling_ratio/max": 1.7320148944854736,
      "sampling/importance_sampling_ratio/mean": 1.0001366138458252,
      "sampling/importance_sampling_ratio/min": 0.5123909115791321,
      "sampling/sampling_logp_difference/max": 0.6686674356460571,
      "sampling/sampling_logp_difference/mean": 0.014301672577857971,
      "step": 3,
      "step_time": 76.45311136299915
    },
    {
      "clip_ratio/high_max": 0.0063213135581463575,
      "clip_ratio/high_mean": 0.003855101269437,
      "clip_ratio/low_mean": 0.0049675037153065205,
      "clip_ratio/low_min": 0.0006944444612599909,
      "clip_ratio/region_mean": 0.00882260489743203,
      "entropy": 0.14919065916910768,
      "epoch": 4e-05,
      "grad_norm": 1.35505211353302,
      "kl": 0.0014064895312912995,
      "learning_rate": 8.571428571428573e-06,
      "loss": -0.2073,
      "step": 4,
      "step_time": 34.60768302199995
    },
    {
      "clip_ratio/high_max": 0.007777874590829015,
      "clip_ratio/high_mean": 0.0042565843323245645,
      "clip_ratio/low_mean": 0.003121181536698714,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0073777658108156174,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11443.0,
      "completions/max_terminated_length": 11443.0,
      "completions/mean_length": 8618.625,
      "completions/mean_terminated_length": 8618.625,
      "completions/min_length": 336.0,
      "completions/min_terminated_length": 336.0,
      "entropy": 0.11723149428144097,
      "epoch": 5e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2567578554153442,
      "kl": 0.0012220569333294407,
      "learning_rate": 1.1428571428571429e-05,
      "loss": 0.14,
      "num_tokens": 917591.0,
      "reward": -0.0526789128780365,
      "reward_std": 0.7260563373565674,
      "rewards/rollout_eval_reward_func/mean": 0.24098068475723267,
      "rewards/rollout_eval_reward_func/std": 0.30969831347465515,
      "rewards/rollout_reward_func/mean": -0.0526789128780365,
      "rewards/rollout_reward_func/std": 0.7283722758293152,
      "sampling/importance_sampling_ratio/max": 1.8090978860855103,
      "sampling/importance_sampling_ratio/mean": 0.9998831748962402,
      "sampling/importance_sampling_ratio/min": 0.46312808990478516,
      "sampling/sampling_logp_difference/max": 0.7697516679763794,
      "sampling/sampling_logp_difference/mean": 0.012965342029929161,
      "step": 5,
      "step_time": 76.55231227500053
    },
    {
      "clip_ratio/high_max": 0.006731559580657631,
      "clip_ratio/high_mean": 0.0033657797903288156,
      "clip_ratio/low_mean": 0.0034470450482331216,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006812824809458107,
      "entropy": 0.1164796226657927,
      "epoch": 6e-05,
      "grad_norm": 1.2304322719573975,
      "kl": 0.001537940081107081,
      "learning_rate": 1.4285714285714285e-05,
      "loss": 0.1411,
      "step": 6,
      "step_time": 34.486522669000806
    },
    {
      "clip_ratio/high_max": 0.004329371266067028,
      "clip_ratio/high_mean": 0.002164685633033514,
      "clip_ratio/low_mean": 0.0018710909644141793,
      "clip_ratio/low_min": 0.0007183908019214869,
      "clip_ratio/region_mean": 0.004035776626551524,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11026.0,
      "completions/max_terminated_length": 11026.0,
      "completions/mean_length": 8904.9375,
      "completions/mean_terminated_length": 8904.9375,
      "completions/min_length": 1870.0,
      "completions/min_terminated_length": 1870.0,
      "entropy": 0.11141919903457165,
      "epoch": 7e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0392634868621826,
      "kl": 0.0028663520533882547,
      "learning_rate": 1.7142857142857145e-05,
      "loss": 0.2956,
      "num_tokens": 1231697.0,
      "reward": 0.06170085445046425,
      "reward_std": 0.6284129619598389,
      "rewards/rollout_eval_reward_func/mean": 0.1812753975391388,
      "rewards/rollout_eval_reward_func/std": 0.29651859402656555,
      "rewards/rollout_reward_func/mean": 0.06170085445046425,
      "rewards/rollout_reward_func/std": 0.6143175363540649,
      "sampling/importance_sampling_ratio/max": 3.0,
      "sampling/importance_sampling_ratio/mean": 1.0020291805267334,
      "sampling/importance_sampling_ratio/min": 0.4785745441913605,
      "sampling/sampling_logp_difference/max": 1.289139747619629,
      "sampling/sampling_logp_difference/mean": 0.012369468808174133,
      "step": 7,
      "step_time": 79.23396538400038
    },
    {
      "clip_ratio/high_max": 0.004530098114628345,
      "clip_ratio/high_mean": 0.0022650490573141724,
      "clip_ratio/low_mean": 0.0062050665146671236,
      "clip_ratio/low_min": 0.0007183908019214869,
      "clip_ratio/region_mean": 0.008470115601085126,
      "entropy": 0.10932181030511856,
      "epoch": 8e-05,
      "grad_norm": 1.4861699342727661,
      "kl": 0.004455661201063776,
      "learning_rate": 2e-05,
      "loss": 0.291,
      "step": 8,
      "step_time": 33.553139639999245
    },
    {
      "clip_ratio/high_max": 0.006702587008476257,
      "clip_ratio/high_mean": 0.00395225512329489,
      "clip_ratio/low_mean": 0.001561393088195473,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005513648211490363,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11738.0,
      "completions/max_terminated_length": 11738.0,
      "completions/mean_length": 7849.21875,
      "completions/mean_terminated_length": 7849.21875,
      "completions/min_length": 323.0,
      "completions/min_terminated_length": 323.0,
      "entropy": 0.10142783727496862,
      "epoch": 9e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1208703517913818,
      "kl": 0.005338215534720803,
      "learning_rate": 2.2857142857142858e-05,
      "loss": -0.0444,
      "num_tokens": 1511511.0,
      "reward": -0.1137353852391243,
      "reward_std": 0.6742830872535706,
      "rewards/rollout_eval_reward_func/mean": 0.21138212084770203,
      "rewards/rollout_eval_reward_func/std": 0.2764902412891388,
      "rewards/rollout_reward_func/mean": -0.1137353852391243,
      "rewards/rollout_reward_func/std": 0.6757553219795227,
      "sampling/importance_sampling_ratio/max": 1.9115883111953735,
      "sampling/importance_sampling_ratio/mean": 0.9999130964279175,
      "sampling/importance_sampling_ratio/min": 0.6430048942565918,
      "sampling/sampling_logp_difference/max": 0.6479344367980957,
      "sampling/sampling_logp_difference/mean": 0.009747568517923355,
      "step": 9,
      "step_time": 78.92596304499875
    },
    {
      "clip_ratio/high_max": 0.014946904906537384,
      "clip_ratio/high_mean": 0.008911944547435269,
      "clip_ratio/low_mean": 0.008717332733795047,
      "clip_ratio/low_min": 0.0007022471982054412,
      "clip_ratio/region_mean": 0.017629277397645637,
      "entropy": 0.09894524328410625,
      "epoch": 0.0001,
      "grad_norm": 0.8779318332672119,
      "kl": 0.012525228099548258,
      "learning_rate": 2.5714285714285714e-05,
      "loss": -0.0474,
      "step": 10,
      "step_time": 33.86218772000029
    },
    {
      "clip_ratio/high_max": 0.005196896672714502,
      "clip_ratio/high_mean": 0.002598448336357251,
      "clip_ratio/low_mean": 0.00565887795528397,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00825732629164122,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11663.0,
      "completions/max_terminated_length": 11663.0,
      "completions/mean_length": 9176.8125,
      "completions/mean_terminated_length": 9176.8125,
      "completions/min_length": 2189.0,
      "completions/min_terminated_length": 2189.0,
      "entropy": 0.1143840029835701,
      "epoch": 0.00011,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9076919555664062,
      "kl": 0.02556461405765731,
      "learning_rate": 2.857142857142857e-05,
      "loss": 0.1457,
      "num_tokens": 1833833.0,
      "reward": -0.07531464844942093,
      "reward_std": 0.6547598838806152,
      "rewards/rollout_eval_reward_func/mean": 0.1659044623374939,
      "rewards/rollout_eval_reward_func/std": 0.27956825494766235,
      "rewards/rollout_reward_func/mean": -0.07531464844942093,
      "rewards/rollout_reward_func/std": 0.6382541060447693,
      "sampling/importance_sampling_ratio/max": 2.9357423782348633,
      "sampling/importance_sampling_ratio/mean": 0.9987922310829163,
      "sampling/importance_sampling_ratio/min": 0.4146113395690918,
      "sampling/sampling_logp_difference/max": 1.0769603252410889,
      "sampling/sampling_logp_difference/mean": 0.01623542606830597,
      "step": 11,
      "step_time": 85.64898062899738
    },
    {
      "clip_ratio/high_max": 0.007925633108243346,
      "clip_ratio/high_mean": 0.003962816554121673,
      "clip_ratio/low_mean": 0.013630853296490386,
      "clip_ratio/low_min": 0.004900663043372333,
      "clip_ratio/region_mean": 0.01759366990881972,
      "entropy": 0.11427765572443604,
      "epoch": 0.00012,
      "grad_norm": 1.6511975526809692,
      "kl": 0.048679544823244214,
      "learning_rate": 3.142857142857143e-05,
      "loss": 0.142,
      "step": 12,
      "step_time": 34.99860805099979
    },
    {
      "clip_ratio/high_max": 0.003243650193326175,
      "clip_ratio/high_mean": 0.0016218250966630876,
      "clip_ratio/low_mean": 0.0024501511070411652,
      "clip_ratio/low_min": 0.0006944444612599909,
      "clip_ratio/region_mean": 0.004071976203704253,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11007.0,
      "completions/max_terminated_length": 11007.0,
      "completions/mean_length": 8355.625,
      "completions/mean_terminated_length": 8355.625,
      "completions/min_length": 874.0,
      "completions/min_terminated_length": 874.0,
      "entropy": 0.11024783831089735,
      "epoch": 0.00013,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.4035089015960693,
      "kl": 0.08716274873586372,
      "learning_rate": 3.428571428571429e-05,
      "loss": 0.1218,
      "num_tokens": 2130373.0,
      "reward": 0.020019814372062683,
      "reward_std": 0.6314284801483154,
      "rewards/rollout_eval_reward_func/mean": 0.21633638441562653,
      "rewards/rollout_eval_reward_func/std": 0.31151828169822693,
      "rewards/rollout_reward_func/mean": 0.020019814372062683,
      "rewards/rollout_reward_func/std": 0.689924418926239,
      "sampling/importance_sampling_ratio/max": 3.0,
      "sampling/importance_sampling_ratio/mean": 1.0034040212631226,
      "sampling/importance_sampling_ratio/min": 0.20111392438411713,
      "sampling/sampling_logp_difference/max": 1.6038837432861328,
      "sampling/sampling_logp_difference/mean": 0.02121809497475624,
      "step": 13,
      "step_time": 86.70206161200258
    },
    {
      "clip_ratio/high_max": 0.009814315650146455,
      "clip_ratio/high_mean": 0.005678762652678415,
      "clip_ratio/low_mean": 0.012552198284538463,
      "clip_ratio/low_min": 0.004237600835040212,
      "clip_ratio/region_mean": 0.01823096099542454,
      "entropy": 0.10833629406988621,
      "epoch": 0.00014,
      "grad_norm": 2.254364490509033,
      "kl": 0.07821251518907957,
      "learning_rate": 3.7142857142857143e-05,
      "loss": 0.1127,
      "step": 14,
      "step_time": 33.7453661569989
    },
    {
      "clip_ratio/high_max": 0.004190173640381545,
      "clip_ratio/high_mean": 0.0020950868201907724,
      "clip_ratio/low_mean": 0.0028092548600398004,
      "clip_ratio/low_min": 0.0007022471982054412,
      "clip_ratio/region_mean": 0.004904341680230573,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10761.0,
      "completions/max_terminated_length": 10761.0,
      "completions/mean_length": 8177.6875,
      "completions/mean_terminated_length": 8177.6875,
      "completions/min_length": 811.0,
      "completions/min_terminated_length": 811.0,
      "entropy": 0.10465786419808865,
      "epoch": 0.00015,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6561901569366455,
      "kl": 0.1072866928880103,
      "learning_rate": 4e-05,
      "loss": 0.356,
      "num_tokens": 2420931.0,
      "reward": 0.11278900504112244,
      "reward_std": 0.6452403664588928,
      "rewards/rollout_eval_reward_func/mean": 0.23056402802467346,
      "rewards/rollout_eval_reward_func/std": 0.3142653703689575,
      "rewards/rollout_reward_func/mean": 0.11278900504112244,
      "rewards/rollout_reward_func/std": 0.6579005718231201,
      "sampling/importance_sampling_ratio/max": 2.278707504272461,
      "sampling/importance_sampling_ratio/mean": 0.9995414018630981,
      "sampling/importance_sampling_ratio/min": 0.18554040789604187,
      "sampling/sampling_logp_difference/max": 1.6844825744628906,
      "sampling/sampling_logp_difference/mean": 0.023679137229919434,
      "step": 15,
      "step_time": 84.81073227599518
    },
    {
      "clip_ratio/high_max": 0.00427410431439057,
      "clip_ratio/high_mean": 0.002137052157195285,
      "clip_ratio/low_mean": 0.013172850711271167,
      "clip_ratio/low_min": 0.00423778424737975,
      "clip_ratio/region_mean": 0.015309902926674113,
      "entropy": 0.1049683466553688,
      "epoch": 0.00016,
      "grad_norm": 1.5131983757019043,
      "kl": 0.12803897575940937,
      "learning_rate": 4.2857142857142856e-05,
      "loss": 0.3499,
      "step": 16,
      "step_time": 32.4860796890016
    },
    {
      "clip_ratio/high_max": 0.006543069262988865,
      "clip_ratio/high_mean": 0.003775566816329956,
      "clip_ratio/low_mean": 0.0031609196157660335,
      "clip_ratio/low_min": 0.0006944444612599909,
      "clip_ratio/region_mean": 0.00693648643209599,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11153.0,
      "completions/max_terminated_length": 11153.0,
      "completions/mean_length": 8529.6875,
      "completions/mean_terminated_length": 8529.6875,
      "completions/min_length": 330.0,
      "completions/min_terminated_length": 330.0,
      "entropy": 0.10206164792180061,
      "epoch": 0.00017,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3023760318756104,
      "kl": 0.09016790508758277,
      "learning_rate": 4.5714285714285716e-05,
      "loss": -0.055,
      "num_tokens": 2722874.0,
      "reward": 0.0452897846698761,
      "reward_std": 0.5471868515014648,
      "rewards/rollout_eval_reward_func/mean": 0.16552338004112244,
      "rewards/rollout_eval_reward_func/std": 0.2769395411014557,
      "rewards/rollout_reward_func/mean": 0.0452897846698761,
      "rewards/rollout_reward_func/std": 0.5707083344459534,
      "sampling/importance_sampling_ratio/max": 1.9813306331634521,
      "sampling/importance_sampling_ratio/mean": 0.9965455532073975,
      "sampling/importance_sampling_ratio/min": 0.1743103712797165,
      "sampling/sampling_logp_difference/max": 1.746917724609375,
      "sampling/sampling_logp_difference/mean": 0.021249916404485703,
      "step": 17,
      "step_time": 93.31380597800035
    },
    {
      "clip_ratio/high_max": 0.013804474321659654,
      "clip_ratio/high_mean": 0.008459640172077343,
      "clip_ratio/low_mean": 0.009875382616883144,
      "clip_ratio/low_min": 0.002192793588619679,
      "clip_ratio/region_mean": 0.018335022643441334,
      "entropy": 0.10283505916595459,
      "epoch": 0.00018,
      "grad_norm": 1.0806056261062622,
      "kl": 0.11839380749734119,
      "learning_rate": 4.8571428571428576e-05,
      "loss": -0.0586,
      "step": 18,
      "step_time": 32.5428760199984
    },
    {
      "clip_ratio/high_max": 0.004807963909115642,
      "clip_ratio/high_mean": 0.002403981954557821,
      "clip_ratio/low_mean": 0.004428166459547356,
      "clip_ratio/low_min": 0.0006944444612599909,
      "clip_ratio/region_mean": 0.006832148414105177,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10399.0,
      "completions/max_terminated_length": 10399.0,
      "completions/mean_length": 7525.5625,
      "completions/mean_terminated_length": 7525.5625,
      "completions/min_length": 1309.0,
      "completions/min_terminated_length": 1309.0,
      "entropy": 0.10726649826392531,
      "epoch": 0.00019,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3821837902069092,
      "kl": 0.09694959409534931,
      "learning_rate": 5.142857142857143e-05,
      "loss": 0.2508,
      "num_tokens": 2992614.0,
      "reward": 0.22085225582122803,
      "reward_std": 0.7439246773719788,
      "rewards/rollout_eval_reward_func/mean": 0.32672765851020813,
      "rewards/rollout_eval_reward_func/std": 0.3242030739784241,
      "rewards/rollout_reward_func/mean": 0.22085225582122803,
      "rewards/rollout_reward_func/std": 0.7110569477081299,
      "sampling/importance_sampling_ratio/max": 3.0,
      "sampling/importance_sampling_ratio/mean": 1.0050089359283447,
      "sampling/importance_sampling_ratio/min": 0.2929533123970032,
      "sampling/sampling_logp_difference/max": 1.916597604751587,
      "sampling/sampling_logp_difference/mean": 0.022352777421474457,
      "step": 19,
      "step_time": 89.20460286800153
    },
    {
      "clip_ratio/high_max": 0.011035054223611951,
      "clip_ratio/high_mean": 0.0055175271118059754,
      "clip_ratio/low_mean": 0.018963277572765946,
      "clip_ratio/low_min": 0.0055555556900799274,
      "clip_ratio/region_mean": 0.024480804742779583,
      "entropy": 0.1047364235855639,
      "epoch": 0.0002,
      "grad_norm": 0.9138187170028687,
      "kl": 0.10579214652534574,
      "learning_rate": 5.428571428571428e-05,
      "loss": 0.2408,
      "step": 20,
      "step_time": 30.517773796002075
    },
    {
      "clip_ratio/high_max": 0.004395612399093807,
      "clip_ratio/high_mean": 0.0028961520874872804,
      "clip_ratio/low_mean": 0.0034880968159995973,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006384248874383047,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11335.0,
      "completions/max_terminated_length": 11335.0,
      "completions/mean_length": 8358.6875,
      "completions/mean_terminated_length": 8358.6875,
      "completions/min_length": 749.0,
      "completions/min_terminated_length": 749.0,
      "entropy": 0.12294450332410634,
      "epoch": 0.00021,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3461740016937256,
      "kl": 0.19072263734415174,
      "learning_rate": 5.714285714285714e-05,
      "loss": -0.0037,
      "num_tokens": 3288905.0,
      "reward": -0.1559383124113083,
      "reward_std": 0.5600723028182983,
      "rewards/rollout_eval_reward_func/mean": 0.14430895447731018,
      "rewards/rollout_eval_reward_func/std": 0.2627376317977905,
      "rewards/rollout_reward_func/mean": -0.1559383124113083,
      "rewards/rollout_reward_func/std": 0.5887359380722046,
      "sampling/importance_sampling_ratio/max": 2.2654411792755127,
      "sampling/importance_sampling_ratio/mean": 0.9976513385772705,
      "sampling/importance_sampling_ratio/min": 0.20272883772850037,
      "sampling/sampling_logp_difference/max": 1.595885992050171,
      "sampling/sampling_logp_difference/mean": 0.02302836999297142,
      "step": 21,
      "step_time": 97.79121274199952
    },
    {
      "clip_ratio/high_max": 0.015567402355372906,
      "clip_ratio/high_mean": 0.009180392837151885,
      "clip_ratio/low_mean": 0.017762881761882454,
      "clip_ratio/low_min": 0.006345431203953922,
      "clip_ratio/region_mean": 0.026943274919176474,
      "entropy": 0.12321826652623713,
      "epoch": 0.00022,
      "grad_norm": 1.64430570602417,
      "kl": 0.269085428211838,
      "learning_rate": 6e-05,
      "loss": -0.0084,
      "step": 22,
      "step_time": 32.807887424998626
    },
    {
      "clip_ratio/high_max": 0.003910088969860226,
      "clip_ratio/high_mean": 0.001955044484930113,
      "clip_ratio/low_mean": 0.002580859523732215,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004535904008662328,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11344.0,
      "completions/max_terminated_length": 11344.0,
      "completions/mean_length": 8706.71875,
      "completions/mean_terminated_length": 8706.71875,
      "completions/min_length": 1846.0,
      "completions/min_terminated_length": 1846.0,
      "entropy": 0.1066858924459666,
      "epoch": 0.00023,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.9362024068832397,
      "kl": 0.12137257889844477,
      "learning_rate": 6.285714285714286e-05,
      "loss": 0.2385,
      "num_tokens": 3596056.0,
      "reward": 0.12735633552074432,
      "reward_std": 0.6423711776733398,
      "rewards/rollout_eval_reward_func/mean": 0.2489837408065796,
      "rewards/rollout_eval_reward_func/std": 0.3152793347835541,
      "rewards/rollout_reward_func/mean": 0.12735633552074432,
      "rewards/rollout_reward_func/std": 0.7003796696662903,
      "sampling/importance_sampling_ratio/max": 2.351552963256836,
      "sampling/importance_sampling_ratio/mean": 1.0003013610839844,
      "sampling/importance_sampling_ratio/min": 0.3779045045375824,
      "sampling/sampling_logp_difference/max": 0.9731137752532959,
      "sampling/sampling_logp_difference/mean": 0.016432534903287888,
      "step": 23,
      "step_time": 99.05041999199784
    },
    {
      "clip_ratio/high_max": 0.011161994189023972,
      "clip_ratio/high_mean": 0.0066947695158887655,
      "clip_ratio/low_mean": 0.009474401478655636,
      "clip_ratio/low_min": 0.0021156331058591604,
      "clip_ratio/region_mean": 0.01616917096544057,
      "entropy": 0.11043372261337936,
      "epoch": 0.00024,
      "grad_norm": 1.812207818031311,
      "kl": 0.09031509491614997,
      "learning_rate": 6.571428571428571e-05,
      "loss": 0.2393,
      "step": 24,
      "step_time": 32.823300022000694
    },
    {
      "clip_ratio/high_max": 0.006489152496214956,
      "clip_ratio/high_mean": 0.003244576248107478,
      "clip_ratio/low_mean": 0.0018077631539199501,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005052339431131259,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10988.0,
      "completions/max_terminated_length": 10988.0,
      "completions/mean_length": 8457.65625,
      "completions/mean_terminated_length": 8457.65625,
      "completions/min_length": 1525.0,
      "completions/min_terminated_length": 1525.0,
      "entropy": 0.11516976961866021,
      "epoch": 0.00025,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3577765226364136,
      "kl": 0.1303024498047307,
      "learning_rate": 6.857142857142858e-05,
      "loss": 0.1265,
      "num_tokens": 3894768.0,
      "reward": 0.20315799117088318,
      "reward_std": 0.6316713094711304,
      "rewards/rollout_eval_reward_func/mean": 0.240091472864151,
      "rewards/rollout_eval_reward_func/std": 0.31059667468070984,
      "rewards/rollout_reward_func/mean": 0.20315799117088318,
      "rewards/rollout_reward_func/std": 0.6295517683029175,
      "sampling/importance_sampling_ratio/max": 2.256138801574707,
      "sampling/importance_sampling_ratio/mean": 0.9993441104888916,
      "sampling/importance_sampling_ratio/min": 0.2282668501138687,
      "sampling/sampling_logp_difference/max": 1.477239966392517,
      "sampling/sampling_logp_difference/mean": 0.01688190922141075,
      "step": 25,
      "step_time": 96.7960174280006
    },
    {
      "clip_ratio/high_max": 0.016165015986189246,
      "clip_ratio/high_mean": 0.010140184662304819,
      "clip_ratio/low_mean": 0.027886587660759687,
      "clip_ratio/low_min": 0.013233458623290062,
      "clip_ratio/region_mean": 0.038026772264856845,
      "entropy": 0.11189563758671284,
      "epoch": 0.00026,
      "grad_norm": 2.0246684551239014,
      "kl": 0.2296465744730085,
      "learning_rate": 7.142857142857143e-05,
      "loss": 0.1164,
      "step": 26,
      "step_time": 32.81865678699978
    },
    {
      "clip_ratio/high_max": 0.0038667930057272315,
      "clip_ratio/high_mean": 0.0019333965028636158,
      "clip_ratio/low_mean": 0.003156299644615501,
      "clip_ratio/low_min": 0.0014046717551536858,
      "clip_ratio/region_mean": 0.005089696147479117,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11043.0,
      "completions/max_terminated_length": 11043.0,
      "completions/mean_length": 8288.09375,
      "completions/mean_terminated_length": 8288.09375,
      "completions/min_length": 873.0,
      "completions/min_terminated_length": 873.0,
      "entropy": 0.1172712086699903,
      "epoch": 0.00027,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0188708305358887,
      "kl": 0.14066953863948584,
      "learning_rate": 7.428571428571429e-05,
      "loss": 0.019,
      "num_tokens": 4187980.0,
      "reward": 0.3152431547641754,
      "reward_std": 0.6300436854362488,
      "rewards/rollout_eval_reward_func/mean": 0.31758129596710205,
      "rewards/rollout_eval_reward_func/std": 0.32922983169555664,
      "rewards/rollout_reward_func/mean": 0.3152431547641754,
      "rewards/rollout_reward_func/std": 0.6763298511505127,
      "sampling/importance_sampling_ratio/max": 3.0,
      "sampling/importance_sampling_ratio/mean": 0.9995622634887695,
      "sampling/importance_sampling_ratio/min": 0.4497600793838501,
      "sampling/sampling_logp_difference/max": 1.9223380088806152,
      "sampling/sampling_logp_difference/mean": 0.017684465274214745,
      "step": 27,
      "step_time": 96.53867048800112
    },
    {
      "clip_ratio/high_max": 0.01634173752972856,
      "clip_ratio/high_mean": 0.00923296029213816,
      "clip_ratio/low_mean": 0.011995688633760437,
      "clip_ratio/low_min": 0.0042134831310249865,
      "clip_ratio/region_mean": 0.021228648955002427,
      "entropy": 0.11727899359539151,
      "epoch": 0.00028,
      "grad_norm": 1.0265206098556519,
      "kl": 0.1646709272172302,
      "learning_rate": 7.714285714285715e-05,
      "loss": 0.0122,
      "step": 28,
      "step_time": 32.412335121999604
    },
    {
      "clip_ratio/high_max": 0.0030616914154961705,
      "clip_ratio/high_mean": 0.0015308457077480853,
      "clip_ratio/low_mean": 0.0017557066748850048,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003286552324425429,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11171.0,
      "completions/max_terminated_length": 11171.0,
      "completions/mean_length": 7933.9375,
      "completions/mean_terminated_length": 7933.9375,
      "completions/min_length": 1855.0,
      "completions/min_terminated_length": 1855.0,
      "entropy": 0.10820061061531305,
      "epoch": 0.00029,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.078338623046875,
      "kl": 0.12938267178833485,
      "learning_rate": 8e-05,
      "loss": 0.1072,
      "num_tokens": 4469763.0,
      "reward": 0.19558671116828918,
      "reward_std": 0.6997864842414856,
      "rewards/rollout_eval_reward_func/mean": 0.2963668704032898,
      "rewards/rollout_eval_reward_func/std": 0.3109610080718994,
      "rewards/rollout_reward_func/mean": 0.19558671116828918,
      "rewards/rollout_reward_func/std": 0.6721439361572266,
      "sampling/importance_sampling_ratio/max": 1.8601394891738892,
      "sampling/importance_sampling_ratio/mean": 1.0000590085983276,
      "sampling/importance_sampling_ratio/min": 0.5378966927528381,
      "sampling/sampling_logp_difference/max": 0.6206514835357666,
      "sampling/sampling_logp_difference/mean": 0.01655469462275505,
      "step": 29,
      "step_time": 93.66265234299863
    },
    {
      "clip_ratio/high_max": 0.017240845831111073,
      "clip_ratio/high_mean": 0.009188604657538235,
      "clip_ratio/low_mean": 0.018311504449229687,
      "clip_ratio/low_min": 0.004907927650492638,
      "clip_ratio/region_mean": 0.027500109281390905,
      "entropy": 0.10658967308700085,
      "epoch": 0.0003,
      "grad_norm": 0.5991150140762329,
      "kl": 0.14486833242699504,
      "learning_rate": 8.285714285714287e-05,
      "loss": 0.0961,
      "step": 30,
      "step_time": 32.02409046399953
    },
    {
      "clip_ratio/high_max": 0.007639755902346224,
      "clip_ratio/high_mean": 0.003819877951173112,
      "clip_ratio/low_mean": 0.0032558340462855995,
      "clip_ratio/low_min": 0.000811688310932368,
      "clip_ratio/region_mean": 0.007075712055666372,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11207.0,
      "completions/max_terminated_length": 11207.0,
      "completions/mean_length": 7446.4375,
      "completions/mean_terminated_length": 7446.4375,
      "completions/min_length": 1941.0,
      "completions/min_terminated_length": 1941.0,
      "entropy": 0.1075986991636455,
      "epoch": 0.00031,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3741053342819214,
      "kl": 0.1601699753664434,
      "learning_rate": 8.571428571428571e-05,
      "loss": 0.3119,
      "num_tokens": 4736188.0,
      "reward": 0.3060583770275116,
      "reward_std": 0.7248314619064331,
      "rewards/rollout_eval_reward_func/mean": 0.3489583134651184,
      "rewards/rollout_eval_reward_func/std": 0.3138476312160492,
      "rewards/rollout_reward_func/mean": 0.3060583770275116,
      "rewards/rollout_reward_func/std": 0.7231479287147522,
      "sampling/importance_sampling_ratio/max": 2.0959339141845703,
      "sampling/importance_sampling_ratio/mean": 1.0005576610565186,
      "sampling/importance_sampling_ratio/min": 0.3962559998035431,
      "sampling/sampling_logp_difference/max": 0.9256948232650757,
      "sampling/sampling_logp_difference/mean": 0.015314233489334583,
      "step": 31,
      "step_time": 89.359079396997
    },
    {
      "clip_ratio/high_max": 0.018800459045451134,
      "clip_ratio/high_mean": 0.010931086115306243,
      "clip_ratio/low_mean": 0.014964523201342672,
      "clip_ratio/low_min": 0.002763190073892474,
      "clip_ratio/region_mean": 0.025895609316648915,
      "entropy": 0.11107803648337722,
      "epoch": 0.00032,
      "grad_norm": 0.8436726927757263,
      "kl": 0.17236854787915945,
      "learning_rate": 8.857142857142857e-05,
      "loss": 0.2984,
      "step": 32,
      "step_time": 32.26129659800063
    },
    {
      "clip_ratio/high_max": 0.005184833600651473,
      "clip_ratio/high_mean": 0.0025924168003257364,
      "clip_ratio/low_mean": 0.0021227904653642327,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004715207265689969,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11234.0,
      "completions/max_terminated_length": 11234.0,
      "completions/mean_length": 8056.90625,
      "completions/mean_terminated_length": 8056.90625,
      "completions/min_length": 834.0,
      "completions/min_terminated_length": 834.0,
      "entropy": 0.10890151979401708,
      "epoch": 0.00033,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.012204647064209,
      "kl": 0.13646527798846364,
      "learning_rate": 9.142857142857143e-05,
      "loss": 0.2208,
      "num_tokens": 5021855.0,
      "reward": 0.28810450434684753,
      "reward_std": 0.6511638164520264,
      "rewards/rollout_eval_reward_func/mean": 0.27172255516052246,
      "rewards/rollout_eval_reward_func/std": 0.3220342993736267,
      "rewards/rollout_reward_func/mean": 0.28810450434684753,
      "rewards/rollout_reward_func/std": 0.6260398030281067,
      "sampling/importance_sampling_ratio/max": 1.8270360231399536,
      "sampling/importance_sampling_ratio/mean": 1.0004476308822632,
      "sampling/importance_sampling_ratio/min": 0.2768998444080353,
      "sampling/sampling_logp_difference/max": 1.2840993404388428,
      "sampling/sampling_logp_difference/mean": 0.012767134234309196,
      "step": 33,
      "step_time": 93.59931743799825
    },
    {
      "clip_ratio/high_max": 0.013980166870169342,
      "clip_ratio/high_mean": 0.008351352356839925,
      "clip_ratio/low_mean": 0.016900451679248363,
      "clip_ratio/low_min": 0.0068813529796898365,
      "clip_ratio/region_mean": 0.025251803919672966,
      "entropy": 0.1073108701966703,
      "epoch": 0.00034,
      "grad_norm": 0.6274716258049011,
      "kl": 0.14035740494728088,
      "learning_rate": 9.428571428571429e-05,
      "loss": 0.2118,
      "step": 34,
      "step_time": 32.919508103002954
    },
    {
      "clip_ratio/high_max": 0.0018894725944846869,
      "clip_ratio/high_mean": 0.0009447362972423434,
      "clip_ratio/low_mean": 0.0021555489802267402,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0031002852774690837,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11099.0,
      "completions/max_terminated_length": 11099.0,
      "completions/mean_length": 8609.5,
      "completions/mean_terminated_length": 8609.5,
      "completions/min_length": 1070.0,
      "completions/min_terminated_length": 1070.0,
      "entropy": 0.10947995679453015,
      "epoch": 0.00035,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9109163284301758,
      "kl": 0.24753205897286534,
      "learning_rate": 9.714285714285715e-05,
      "loss": 0.1542,
      "num_tokens": 5325755.0,
      "reward": 0.4508249759674072,
      "reward_std": 0.5780479907989502,
      "rewards/rollout_eval_reward_func/mean": 0.32621949911117554,
      "rewards/rollout_eval_reward_func/std": 0.3382527828216553,
      "rewards/rollout_reward_func/mean": 0.4508249759674072,
      "rewards/rollout_reward_func/std": 0.5698684453964233,
      "sampling/importance_sampling_ratio/max": 1.7834506034851074,
      "sampling/importance_sampling_ratio/mean": 1.0010955333709717,
      "sampling/importance_sampling_ratio/min": 0.366268515586853,
      "sampling/sampling_logp_difference/max": 1.0043885707855225,
      "sampling/sampling_logp_difference/mean": 0.013199402019381523,
      "step": 35,
      "step_time": 93.34705402800137
    },
    {
      "clip_ratio/high_max": 0.0200398035813123,
      "clip_ratio/high_mean": 0.01071434619370848,
      "clip_ratio/low_mean": 0.010386000329162925,
      "clip_ratio/low_min": 0.0007022471982054412,
      "clip_ratio/region_mean": 0.021100346348248422,
      "entropy": 0.11280779354274273,
      "epoch": 0.00036,
      "grad_norm": 0.8509212732315063,
      "kl": 0.24738536309450865,
      "learning_rate": 0.0001,
      "loss": 0.1443,
      "step": 36,
      "step_time": 33.26461400700191
    },
    {
      "clip_ratio/high_max": 0.0045615925919264555,
      "clip_ratio/high_mean": 0.002661893842741847,
      "clip_ratio/low_mean": 0.001787908433470875,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004449802276212722,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10712.0,
      "completions/max_terminated_length": 10712.0,
      "completions/mean_length": 7822.9375,
      "completions/mean_terminated_length": 7822.9375,
      "completions/min_length": 1061.0,
      "completions/min_terminated_length": 1061.0,
      "entropy": 0.09347328078001738,
      "epoch": 0.00037,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7398293018341064,
      "kl": 0.14611079543828964,
      "learning_rate": 9.918053502751771e-05,
      "loss": 0.0539,
      "num_tokens": 5604207.0,
      "reward": 0.2784110903739929,
      "reward_std": 0.6512280702590942,
      "rewards/rollout_eval_reward_func/mean": 0.2896341383457184,
      "rewards/rollout_eval_reward_func/std": 0.32505732774734497,
      "rewards/rollout_reward_func/mean": 0.2784110903739929,
      "rewards/rollout_reward_func/std": 0.6340307593345642,
      "sampling/importance_sampling_ratio/max": 1.785902500152588,
      "sampling/importance_sampling_ratio/mean": 0.9995973110198975,
      "sampling/importance_sampling_ratio/min": 0.3934570252895355,
      "sampling/sampling_logp_difference/max": 0.9327834844589233,
      "sampling/sampling_logp_difference/mean": 0.01090280432254076,
      "step": 37,
      "step_time": 89.90687796200018
    },
    {
      "clip_ratio/high_max": 0.014242319972254336,
      "clip_ratio/high_mean": 0.007633455039467663,
      "clip_ratio/low_mean": 0.009654591587604955,
      "clip_ratio/low_min": 0.001453684933949262,
      "clip_ratio/region_mean": 0.017288046510657296,
      "entropy": 0.09478882700204849,
      "epoch": 0.00038,
      "grad_norm": 0.5018179416656494,
      "kl": 0.15739013627171516,
      "learning_rate": 9.675795466159755e-05,
      "loss": 0.0474,
      "step": 38,
      "step_time": 32.40730591600277
    },
    {
      "clip_ratio/high_max": 0.0033184904023073614,
      "clip_ratio/high_mean": 0.0016592452011536807,
      "clip_ratio/low_mean": 0.002118892065482214,
      "clip_ratio/low_min": 0.0007102272938936949,
      "clip_ratio/region_mean": 0.0037781372957397252,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10878.0,
      "completions/max_terminated_length": 10878.0,
      "completions/mean_length": 8825.25,
      "completions/mean_terminated_length": 8825.25,
      "completions/min_length": 2113.0,
      "completions/min_terminated_length": 2113.0,
      "entropy": 0.13806713884696364,
      "epoch": 0.00039,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.113547444343567,
      "kl": 0.20099136885255575,
      "learning_rate": 9.283813728906054e-05,
      "loss": 0.1486,
      "num_tokens": 5915217.0,
      "reward": 0.18237435817718506,
      "reward_std": 0.5858109593391418,
      "rewards/rollout_eval_reward_func/mean": 0.23246951401233673,
      "rewards/rollout_eval_reward_func/std": 0.3161698281764984,
      "rewards/rollout_reward_func/mean": 0.18237435817718506,
      "rewards/rollout_reward_func/std": 0.6235412955284119,
      "sampling/importance_sampling_ratio/max": 1.8600788116455078,
      "sampling/importance_sampling_ratio/mean": 1.0009781122207642,
      "sampling/importance_sampling_ratio/min": 0.6195499897003174,
      "sampling/sampling_logp_difference/max": 0.6206188201904297,
      "sampling/sampling_logp_difference/mean": 0.01361357793211937,
      "step": 39,
      "step_time": 95.42290947900165
    },
    {
      "clip_ratio/high_max": 0.01989291445352137,
      "clip_ratio/high_mean": 0.010865762596949935,
      "clip_ratio/low_mean": 0.01921083332854323,
      "clip_ratio/low_min": 0.004245770338457078,
      "clip_ratio/region_mean": 0.030076596303842962,
      "entropy": 0.13848060183227062,
      "epoch": 0.0004,
      "grad_norm": 0.8446061611175537,
      "kl": 0.22338007856160402,
      "learning_rate": 8.759239773845718e-05,
      "loss": 0.1364,
      "step": 40,
      "step_time": 33.852559433999886
    },
    {
      "clip_ratio/high_max": 0.0030503973248414695,
      "clip_ratio/high_mean": 0.0015251986624207348,
      "clip_ratio/low_mean": 0.0010654326470103115,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0025906313094310462,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11107.0,
      "completions/max_terminated_length": 11107.0,
      "completions/mean_length": 8257.21875,
      "completions/mean_terminated_length": 8257.21875,
      "completions/min_length": 2666.0,
      "completions/min_terminated_length": 2666.0,
      "entropy": 0.11340034916065633,
      "epoch": 0.00041,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0817996263504028,
      "kl": 0.20612840354442596,
      "learning_rate": 8.125000000000001e-05,
      "loss": 0.3665,
      "num_tokens": 6207607.0,
      "reward": 0.42999449372291565,
      "reward_std": 0.5903699994087219,
      "rewards/rollout_eval_reward_func/mean": 0.33269816637039185,
      "rewards/rollout_eval_reward_func/std": 0.343418687582016,
      "rewards/rollout_reward_func/mean": 0.42999449372291565,
      "rewards/rollout_reward_func/std": 0.5738593935966492,
      "sampling/importance_sampling_ratio/max": 1.4319037199020386,
      "sampling/importance_sampling_ratio/mean": 0.9990136027336121,
      "sampling/importance_sampling_ratio/min": 0.4935825765132904,
      "sampling/sampling_logp_difference/max": 0.7060651779174805,
      "sampling/sampling_logp_difference/mean": 0.011390198953449726,
      "step": 41,
      "step_time": 91.59171992200027
    },
    {
      "clip_ratio/high_max": 0.01730476424563676,
      "clip_ratio/high_mean": 0.009731545054819435,
      "clip_ratio/low_mean": 0.01579960942035541,
      "clip_ratio/low_min": 0.006750052853021771,
      "clip_ratio/region_mean": 0.025531154591590166,
      "entropy": 0.11443871166557074,
      "epoch": 0.00042,
      "grad_norm": 0.7992480397224426,
      "kl": 0.27106572315096855,
      "learning_rate": 7.408813728906053e-05,
      "loss": 0.3537,
      "step": 42,
      "step_time": 33.60049366600106
    },
    {
      "clip_ratio/high_max": 0.0035054103354923427,
      "clip_ratio/high_mean": 0.0017527051677461714,
      "clip_ratio/low_mean": 0.0010694226948544383,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0028221279208082706,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10896.0,
      "completions/max_terminated_length": 10896.0,
      "completions/mean_length": 7894.15625,
      "completions/mean_terminated_length": 7894.15625,
      "completions/min_length": 2890.0,
      "completions/min_terminated_length": 2890.0,
      "entropy": 0.13558923918753862,
      "epoch": 0.00043,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9851862192153931,
      "kl": 0.24445611471310258,
      "learning_rate": 6.6419817372537e-05,
      "loss": 0.1226,
      "num_tokens": 6488287.0,
      "reward": 0.18586361408233643,
      "reward_std": 0.6816079616546631,
      "rewards/rollout_eval_reward_func/mean": 0.30538618564605713,
      "rewards/rollout_eval_reward_func/std": 0.32574331760406494,
      "rewards/rollout_reward_func/mean": 0.18586361408233643,
      "rewards/rollout_reward_func/std": 0.676216721534729,
      "sampling/importance_sampling_ratio/max": 1.8440579175949097,
      "sampling/importance_sampling_ratio/mean": 1.0000970363616943,
      "sampling/importance_sampling_ratio/min": 0.6517351269721985,
      "sampling/sampling_logp_difference/max": 0.6119685173034668,
      "sampling/sampling_logp_difference/mean": 0.012094310484826565,
      "step": 43,
      "step_time": 92.82438364300197
    },
    {
      "clip_ratio/high_max": 0.010269680351484567,
      "clip_ratio/high_mean": 0.005821653438033536,
      "clip_ratio/low_mean": 0.015070577035658062,
      "clip_ratio/low_min": 0.0048273742431774735,
      "clip_ratio/region_mean": 0.020892230415483937,
      "entropy": 0.13336073094978929,
      "epoch": 0.00044,
      "grad_norm": 0.6729846000671387,
      "kl": 0.2622186541557312,
      "learning_rate": 5.858018262746301e-05,
      "loss": 0.1142,
      "step": 44,
      "step_time": 32.630656069
    },
    {
      "clip_ratio/high_max": 0.006278776447288692,
      "clip_ratio/high_mean": 0.003139388223644346,
      "clip_ratio/low_mean": 0.0014205463230609894,
      "clip_ratio/low_min": 0.0007022471982054412,
      "clip_ratio/region_mean": 0.004559934546705335,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11255.0,
      "completions/max_terminated_length": 11255.0,
      "completions/mean_length": 7849.59375,
      "completions/mean_terminated_length": 7849.59375,
      "completions/min_length": 841.0,
      "completions/min_terminated_length": 841.0,
      "entropy": 0.13749901531264186,
      "epoch": 0.00045,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1264039278030396,
      "kl": 0.22676664125174284,
      "learning_rate": 5.0911862710939485e-05,
      "loss": -0.0333,
      "num_tokens": 6767906.0,
      "reward": 0.15443384647369385,
      "reward_std": 0.693654477596283,
      "rewards/rollout_eval_reward_func/mean": 0.2919207215309143,
      "rewards/rollout_eval_reward_func/std": 0.29990869760513306,
      "rewards/rollout_reward_func/mean": 0.15443384647369385,
      "rewards/rollout_reward_func/std": 0.7213361263275146,
      "sampling/importance_sampling_ratio/max": 1.6071842908859253,
      "sampling/importance_sampling_ratio/mean": 0.9978848099708557,
      "sampling/importance_sampling_ratio/min": 0.48193788528442383,
      "sampling/sampling_logp_difference/max": 0.7299400568008423,
      "sampling/sampling_logp_difference/mean": 0.01217048242688179,
      "step": 45,
      "step_time": 91.51718380699913
    },
    {
      "clip_ratio/high_max": 0.012777922616805881,
      "clip_ratio/high_mean": 0.006918622384546325,
      "clip_ratio/low_mean": 0.014222246245481074,
      "clip_ratio/low_min": 0.005445442744530737,
      "clip_ratio/region_mean": 0.0211408686300274,
      "entropy": 0.1343677118420601,
      "epoch": 0.00046,
      "grad_norm": 0.7688785791397095,
      "kl": 0.24824338778853416,
      "learning_rate": 4.375000000000001e-05,
      "loss": -0.0418,
      "step": 46,
      "step_time": 33.231114288000754
    },
    {
      "clip_ratio/high_max": 0.002997979463543743,
      "clip_ratio/high_mean": 0.0018581851327326149,
      "clip_ratio/low_mean": 0.0012389520416036248,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0030971371743362397,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 10980.0,
      "completions/max_terminated_length": 10980.0,
      "completions/mean_length": 7450.6875,
      "completions/mean_terminated_length": 7450.6875,
      "completions/min_length": 320.0,
      "completions/min_terminated_length": 320.0,
      "entropy": 0.11971927247941494,
      "epoch": 0.00047,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0886417627334595,
      "kl": 0.27644640300422907,
      "learning_rate": 3.740760226154283e-05,
      "loss": 0.0684,
      "num_tokens": 7034802.0,
      "reward": 0.32516512274742126,
      "reward_std": 0.6862237453460693,
      "rewards/rollout_eval_reward_func/mean": 0.3408282399177551,
      "rewards/rollout_eval_reward_func/std": 0.31514763832092285,
      "rewards/rollout_reward_func/mean": 0.32516512274742126,
      "rewards/rollout_reward_func/std": 0.6982549428939819,
      "sampling/importance_sampling_ratio/max": 1.4899945259094238,
      "sampling/importance_sampling_ratio/mean": 0.9988203048706055,
      "sampling/importance_sampling_ratio/min": 0.5982000827789307,
      "sampling/sampling_logp_difference/max": 0.5138299465179443,
      "sampling/sampling_logp_difference/mean": 0.010367423295974731,
      "step": 47,
      "step_time": 92.68640525500086
    },
    {
      "clip_ratio/high_max": 0.005615144735202193,
      "clip_ratio/high_mean": 0.0031981973443180323,
      "clip_ratio/low_mean": 0.008041755587328225,
      "clip_ratio/low_min": 0.0007102272938936949,
      "clip_ratio/region_mean": 0.011239952960750088,
      "entropy": 0.11511751124635339,
      "epoch": 0.00048,
      "grad_norm": 0.677844762802124,
      "kl": 0.2775820675306022,
      "learning_rate": 3.2161862710939476e-05,
      "loss": 0.0609,
      "step": 48,
      "step_time": 32.67402728900106
    },
    {
      "clip_ratio/high_max": 0.00474693434080109,
      "clip_ratio/high_mean": 0.002373467170400545,
      "clip_ratio/low_mean": 0.0003511235991027206,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0027245907695032656,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 11070.0,
      "completions/max_terminated_length": 11070.0,
      "completions/mean_length": 7308.03125,
      "completions/mean_terminated_length": 7308.03125,
      "completions/min_length": 854.0,
      "completions/min_terminated_length": 854.0,
      "entropy": 0.10222014435566962,
      "epoch": 0.00049,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8171713948249817,
      "kl": 0.22370027378201485,
      "learning_rate": 2.8242045338402468e-05,
      "loss": -0.068,
      "num_tokens": 7296677.0,
      "reward": 0.28688520193099976,
      "reward_std": 0.7849241495132446,
      "rewards/rollout_eval_reward_func/mean": 0.37195122241973877,
      "rewards/rollout_eval_reward_func/std": 0.3127909302711487,
      "rewards/rollout_reward_func/mean": 0.28688520193099976,
      "rewards/rollout_reward_func/std": 0.7561083436012268,
      "sampling/importance_sampling_ratio/max": 1.7672066688537598,
      "sampling/importance_sampling_ratio/mean": 1.0014787912368774,
      "sampling/importance_sampling_ratio/min": 0.5275546908378601,
      "sampling/sampling_logp_difference/max": 0.639502763748169,
      "sampling/sampling_logp_difference/mean": 0.011029250919818878,
      "step": 49,
      "step_time": 86.99529032999908
    },
    {
      "clip_ratio/high_max": 0.007674970023799688,
      "clip_ratio/high_mean": 0.004594452795572579,
      "clip_ratio/low_mean": 0.008248247409937903,
      "clip_ratio/low_min": 0.0007183908019214869,
      "clip_ratio/region_mean": 0.012842700205510482,
      "entropy": 0.09808745188638568,
      "epoch": 0.0005,
      "grad_norm": 0.6178692579269409,
      "kl": 0.23346312018111348,
      "learning_rate": 2.5819464972482288e-05,
      "loss": -0.0742,
      "step": 50,
      "step_time": 32.30095837099998
    },
    {
      "epoch": 0.0005,
      "eval_clip_ratio/high_max": 0.0,
      "eval_clip_ratio/high_mean": 0.0,
      "eval_clip_ratio/low_mean": 0.0,
      "eval_clip_ratio/low_min": 0.0,
      "eval_clip_ratio/region_mean": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 10309.35,
      "eval_completions/max_terminated_length": 10309.35,
      "eval_completions/mean_length": 7614.075,
      "eval_completions/mean_terminated_length": 7614.075,
      "eval_completions/min_length": 4457.25,
      "eval_completions/min_terminated_length": 4457.25,
      "eval_entropy": 0.1146618027240038,
      "eval_frac_reward_zero_std": 1.0,
      "eval_kl": 0.2420900955796242,
      "eval_loss": 0.00033982208697125316,
      "eval_num_tokens": 7296677.0,
      "eval_reward": 0.439066107571125,
      "eval_reward_std": 0.0,
      "eval_rewards/rollout_eval_reward_func/mean": 0.3934959270060062,
      "eval_rewards/rollout_eval_reward_func/std": 0.30925278924405575,
      "eval_rewards/rollout_reward_func/mean": 0.439066107571125,
      "eval_rewards/rollout_reward_func/std": 0.643099270761013,
      "eval_runtime": 164.9571,
      "eval_samples_per_second": 0.061,
      "eval_sampling/importance_sampling_ratio/max": 1.303001207113266,
      "eval_sampling/importance_sampling_ratio/mean": 1.0004466891288757,
      "eval_sampling/importance_sampling_ratio/min": 0.7182389438152313,
      "eval_sampling/sampling_logp_difference/max": 0.3734502553939819,
      "eval_sampling/sampling_logp_difference/mean": 0.009935753676109016,
      "eval_steps_per_second": 0.018,
      "step": 50
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 50,
  "num_input_tokens_seen": 7296677,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}