{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.00055,
  "eval_steps": 500,
  "global_step": 55,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.0,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.940144538879395,
      "epoch": 1e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.03423422574996948,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": -0.0013,
      "num_tokens": 35616.0,
      "reward": -0.7051675319671631,
      "reward_std": 0.7764065265655518,
      "rewards/rollout_reward_func/mean": -0.7051675319671631,
      "rewards/rollout_reward_func/std": 0.75037682056427,
      "sampling/importance_sampling_ratio/max": 0.06733503937721252,
      "sampling/importance_sampling_ratio/mean": 0.035891756415367126,
      "sampling/importance_sampling_ratio/min": 0.012922381982207298,
      "sampling/sampling_logp_difference/max": 2.4574475288391113,
      "sampling/sampling_logp_difference/mean": 1.7373101711273193,
      "step": 1,
      "step_time": 6.607899043003272
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 8.940144538879395,
      "epoch": 2e-05,
      "grad_norm": 0.03577549755573273,
      "kl": 0.0,
      "learning_rate": 2.8571428571428575e-07,
      "loss": -0.0013,
      "step": 2,
      "step_time": 2.9063545979988703
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.0,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.965680599212646,
      "epoch": 3e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.017016781494021416,
      "kl": 0.0007822737097740173,
      "learning_rate": 5.714285714285715e-07,
      "loss": -0.0006,
      "num_tokens": 71095.0,
      "reward": -0.9110076427459717,
      "reward_std": 0.6931561231613159,
      "rewards/rollout_reward_func/mean": -0.9110076427459717,
      "rewards/rollout_reward_func/std": 0.6800154447555542,
      "sampling/importance_sampling_ratio/max": 0.06864165514707565,
      "sampling/importance_sampling_ratio/mean": 0.03215230628848076,
      "sampling/importance_sampling_ratio/min": 0.011430883780121803,
      "sampling/sampling_logp_difference/max": 2.474456548690796,
      "sampling/sampling_logp_difference/mean": 1.8041703701019287,
      "step": 3,
      "step_time": 5.5894952089984145
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 8.965598821640015,
      "epoch": 4e-05,
      "grad_norm": 0.01733771711587906,
      "kl": 0.0007491949945688248,
      "learning_rate": 8.571428571428572e-07,
      "loss": -0.0006,
      "step": 4,
      "step_time": 3.4044442560007155
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 16.0,
      "completions/max_terminated_length": 5.0,
      "completions/mean_length": 2.53125,
      "completions/mean_terminated_length": 2.096774101257324,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.834780097007751,
      "epoch": 5e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.02108524739742279,
      "kl": 0.0009654137102188542,
      "learning_rate": 1.142857142857143e-06,
      "loss": -0.0003,
      "num_tokens": 106490.0,
      "reward": -0.5540984869003296,
      "reward_std": 0.8771607279777527,
      "rewards/rollout_reward_func/mean": -0.5540984869003296,
      "rewards/rollout_reward_func/std": 0.8618184924125671,
      "sampling/importance_sampling_ratio/max": 0.07213470339775085,
      "sampling/importance_sampling_ratio/mean": 0.03297191113233566,
      "sampling/importance_sampling_ratio/min": 3.0050444771445584e-11,
      "sampling/sampling_logp_difference/max": 4.576776504516602,
      "sampling/sampling_logp_difference/mean": 1.773134469985962,
      "step": 5,
      "step_time": 6.008008040997083
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 8.827986240386963,
      "epoch": 6e-05,
      "grad_norm": 0.021368548274040222,
      "kl": 0.0009469666983932257,
      "learning_rate": 1.4285714285714286e-06,
      "loss": -0.0004,
      "step": 6,
      "step_time": 2.88994878000085
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 16.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.875,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.796356916427612,
      "epoch": 7e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.013663483783602715,
      "kl": 0.0008234605193138123,
      "learning_rate": 1.7142857142857145e-06,
      "loss": 0.0,
      "num_tokens": 142069.0,
      "reward": -0.8088920712471008,
      "reward_std": 0.7424027323722839,
      "rewards/rollout_reward_func/mean": -0.8088920712471008,
      "rewards/rollout_reward_func/std": 0.7662962675094604,
      "sampling/importance_sampling_ratio/max": 0.057457707822322845,
      "sampling/importance_sampling_ratio/mean": 0.02730659209191799,
      "sampling/importance_sampling_ratio/min": 7.280681058041694e-10,
      "sampling/sampling_logp_difference/max": 4.222927093505859,
      "sampling/sampling_logp_difference/mean": 1.6366889476776123,
      "step": 7,
      "step_time": 5.921918200005166
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 8.786743104457855,
      "epoch": 8e-05,
      "grad_norm": 0.013285573571920395,
      "kl": 0.0009508101793471724,
      "learning_rate": 2.0000000000000003e-06,
      "loss": -0.0,
      "step": 8,
      "step_time": 2.9387520060008683
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.0,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.922944903373718,
      "epoch": 9e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.01735098287463188,
      "kl": 0.0008866805583238602,
      "learning_rate": 2.285714285714286e-06,
      "loss": -0.0002,
      "num_tokens": 176547.0,
      "reward": -0.618694543838501,
      "reward_std": 0.8990023136138916,
      "rewards/rollout_reward_func/mean": -0.618694543838501,
      "rewards/rollout_reward_func/std": 0.8754127621650696,
      "sampling/importance_sampling_ratio/max": 0.06334654986858368,
      "sampling/importance_sampling_ratio/mean": 0.03222377225756645,
      "sampling/importance_sampling_ratio/min": 0.011594683863222599,
      "sampling/sampling_logp_difference/max": 2.4042437076568604,
      "sampling/sampling_logp_difference/mean": 1.7828483581542969,
      "step": 9,
      "step_time": 5.69735375300661
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 8.913637280464172,
      "epoch": 0.0001,
      "grad_norm": 0.017596419900655746,
      "kl": 0.000972965732216835,
      "learning_rate": 2.571428571428571e-06,
      "loss": -0.0002,
      "step": 10,
      "step_time": 3.580516988000454
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5.0,
      "completions/max_terminated_length": 5.0,
      "completions/mean_length": 2.15625,
      "completions/mean_terminated_length": 2.15625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.979163527488708,
      "epoch": 0.00011,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.01395466923713684,
      "kl": 0.0012971882097190246,
      "learning_rate": 2.8571428571428573e-06,
      "loss": -0.0002,
      "num_tokens": 210671.0,
      "reward": -0.6838527917861938,
      "reward_std": 0.7062864899635315,
      "rewards/rollout_reward_func/mean": -0.6838527917861938,
      "rewards/rollout_reward_func/std": 0.7574694752693176,
      "sampling/importance_sampling_ratio/max": 0.06857945024967194,
      "sampling/importance_sampling_ratio/mean": 0.03003668040037155,
      "sampling/importance_sampling_ratio/min": 7.147054475353798e-06,
      "sampling/sampling_logp_difference/max": 4.250937461853027,
      "sampling/sampling_logp_difference/mean": 1.8635720014572144,
      "step": 11,
      "step_time": 6.033825367005193
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 8.966804146766663,
      "epoch": 0.00012,
      "grad_norm": 0.01391494832932949,
      "kl": 0.0018893439264502376,
      "learning_rate": 3.142857142857143e-06,
      "loss": -0.0002,
      "step": 12,
      "step_time": 2.8316939499891305
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5.0,
      "completions/max_terminated_length": 5.0,
      "completions/mean_length": 2.28125,
      "completions/mean_terminated_length": 2.28125,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.810548543930054,
      "epoch": 0.00013,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.014992384240031242,
      "kl": 0.003853602087474428,
      "learning_rate": 3.428571428571429e-06,
      "loss": 0.0001,
      "num_tokens": 245676.0,
      "reward": -0.6364654302597046,
      "reward_std": 0.7521181106567383,
      "rewards/rollout_reward_func/mean": -0.6364654302597046,
      "rewards/rollout_reward_func/std": 0.7526334524154663,
      "sampling/importance_sampling_ratio/max": 0.06722358614206314,
      "sampling/importance_sampling_ratio/mean": 0.03307785466313362,
      "sampling/importance_sampling_ratio/min": 3.5045477488893084e-06,
      "sampling/sampling_logp_difference/max": 4.873165607452393,
      "sampling/sampling_logp_difference/mean": 1.8621257543563843,
      "step": 13,
      "step_time": 5.768471701994713
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 8.802866578102112,
      "epoch": 0.00014,
      "grad_norm": 0.014974371530115604,
      "kl": 0.004468549799639732,
      "learning_rate": 3.7142857142857146e-06,
      "loss": 0.0001,
      "step": 14,
      "step_time": 2.8839251570025226
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 16.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.4375,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.717366337776184,
      "epoch": 0.00015,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.020232753828167915,
      "kl": 0.004971407979610376,
      "learning_rate": 4.000000000000001e-06,
      "loss": -0.0008,
      "num_tokens": 282194.0,
      "reward": -0.7452265024185181,
      "reward_std": 0.7260236144065857,
      "rewards/rollout_reward_func/mean": -0.7452265024185181,
      "rewards/rollout_reward_func/std": 0.7854404449462891,
      "sampling/importance_sampling_ratio/max": 0.084043949842453,
      "sampling/importance_sampling_ratio/mean": 0.03686349838972092,
      "sampling/importance_sampling_ratio/min": 9.963324609785218e-10,
      "sampling/sampling_logp_difference/max": 3.4498603343963623,
      "sampling/sampling_logp_difference/mean": 1.676363468170166,
      "step": 15,
      "step_time": 5.806083219005814
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 8.69223439693451,
      "epoch": 0.00016,
      "grad_norm": 0.020264672115445137,
      "kl": 0.005897294729948044,
      "learning_rate": 4.2857142857142855e-06,
      "loss": -0.0008,
      "step": 16,
      "step_time": 3.649606159000541
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8.0,
      "completions/max_terminated_length": 8.0,
      "completions/mean_length": 2.1875,
      "completions/mean_terminated_length": 2.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.732311606407166,
      "epoch": 0.00017,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.019782064482569695,
      "kl": 0.007078325026668608,
      "learning_rate": 4.571428571428572e-06,
      "loss": -0.0,
      "num_tokens": 317749.0,
      "reward": -0.5659611821174622,
      "reward_std": 0.7136144042015076,
      "rewards/rollout_reward_func/mean": -0.5659611821174622,
      "rewards/rollout_reward_func/std": 0.7692865133285522,
      "sampling/importance_sampling_ratio/max": 0.08927696198225021,
      "sampling/importance_sampling_ratio/mean": 0.034128978848457336,
      "sampling/importance_sampling_ratio/min": 6.115115684224293e-05,
      "sampling/sampling_logp_difference/max": 2.444645404815674,
      "sampling/sampling_logp_difference/mean": 1.729607105255127,
      "step": 17,
      "step_time": 6.096930697000062
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 8.71469098329544,
      "epoch": 0.00018,
      "grad_norm": 0.0198439322412014,
      "kl": 0.00980698294006288,
      "learning_rate": 4.857142857142858e-06,
      "loss": -0.0001,
      "step": 18,
      "step_time": 2.8486052290027146
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 13.0,
      "completions/max_terminated_length": 13.0,
      "completions/mean_length": 2.34375,
      "completions/mean_terminated_length": 2.34375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.700989127159119,
      "epoch": 0.00019,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.02092764899134636,
      "kl": 0.016879421891644597,
      "learning_rate": 5.142857142857142e-06,
      "loss": -0.0005,
      "num_tokens": 353593.0,
      "reward": -0.5766444802284241,
      "reward_std": 0.8734984397888184,
      "rewards/rollout_reward_func/mean": -0.5766444802284241,
      "rewards/rollout_reward_func/std": 0.8666929602622986,
      "sampling/importance_sampling_ratio/max": 0.10328938066959381,
      "sampling/importance_sampling_ratio/mean": 0.0412919819355011,
      "sampling/importance_sampling_ratio/min": 8.264829792770101e-11,
      "sampling/sampling_logp_difference/max": 3.909327507019043,
      "sampling/sampling_logp_difference/mean": 1.7047920227050781,
      "step": 19,
      "step_time": 5.767663798993453
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "entropy": 8.635276675224304,
      "epoch": 0.0002,
      "grad_norm": 0.02117123454809189,
      "kl": 0.022729096352122724,
      "learning_rate": 5.428571428571429e-06,
      "loss": -0.0005,
      "step": 20,
      "step_time": 2.8989755920047173
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 7.0,
      "completions/max_terminated_length": 7.0,
      "completions/mean_length": 2.15625,
      "completions/mean_terminated_length": 2.15625,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.578810691833496,
      "epoch": 0.00021,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.044351667165756226,
      "kl": 0.03346684481948614,
      "learning_rate": 5.7142857142857145e-06,
      "loss": -0.0024,
      "num_tokens": 388691.0,
      "reward": -0.6427146196365356,
      "reward_std": 0.8122553825378418,
      "rewards/rollout_reward_func/mean": -0.6427146196365356,
      "rewards/rollout_reward_func/std": 0.7960423827171326,
      "sampling/importance_sampling_ratio/max": 0.10920954495668411,
      "sampling/importance_sampling_ratio/mean": 0.04724588990211487,
      "sampling/importance_sampling_ratio/min": 2.8349152216833318e-06,
      "sampling/sampling_logp_difference/max": 3.772367477416992,
      "sampling/sampling_logp_difference/mean": 1.6777459383010864,
      "step": 21,
      "step_time": 5.727157995002926
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "entropy": 8.421014785766602,
      "epoch": 0.00022,
      "grad_norm": 0.044636089354753494,
      "kl": 0.047105960082262754,
      "learning_rate": 6e-06,
      "loss": -0.0026,
      "step": 22,
      "step_time": 4.04690631700214
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.0,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.332530975341797,
      "epoch": 0.00023,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0429239459335804,
      "kl": 0.07239408232271671,
      "learning_rate": 6.285714285714286e-06,
      "loss": -0.0028,
      "num_tokens": 424016.0,
      "reward": -0.6825613975524902,
      "reward_std": 0.8769230246543884,
      "rewards/rollout_reward_func/mean": -0.6825613975524902,
      "rewards/rollout_reward_func/std": 0.852479875087738,
      "sampling/importance_sampling_ratio/max": 0.14365191757678986,
      "sampling/importance_sampling_ratio/mean": 0.05794315040111542,
      "sampling/importance_sampling_ratio/min": 0.008735693991184235,
      "sampling/sampling_logp_difference/max": 2.5880439281463623,
      "sampling/sampling_logp_difference/mean": 1.6414165496826172,
      "step": 23,
      "step_time": 5.61804673000006
    },
    {
      "clip_ratio/high_max": 0.21875,
      "clip_ratio/high_mean": 0.109375,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.109375,
      "entropy": 8.144118428230286,
      "epoch": 0.00024,
      "grad_norm": 0.0180932879447937,
      "kl": 0.0962864700704813,
      "learning_rate": 6.571428571428572e-06,
      "loss": -0.0031,
      "step": 24,
      "step_time": 2.8988405260024592
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5.0,
      "completions/max_terminated_length": 5.0,
      "completions/mean_length": 2.1875,
      "completions/mean_terminated_length": 2.1875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 8.125683069229126,
      "epoch": 0.00025,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.03623996675014496,
      "kl": 0.10392077919095755,
      "learning_rate": 6.857142857142858e-06,
      "loss": -0.0041,
      "num_tokens": 459589.0,
      "reward": -0.61258465051651,
      "reward_std": 0.871542751789093,
      "rewards/rollout_reward_func/mean": -0.61258465051651,
      "rewards/rollout_reward_func/std": 0.8524011969566345,
      "sampling/importance_sampling_ratio/max": 0.16312259435653687,
      "sampling/importance_sampling_ratio/mean": 0.06305442750453949,
      "sampling/importance_sampling_ratio/min": 1.7614916032471228e-06,
      "sampling/sampling_logp_difference/max": 4.772340774536133,
      "sampling/sampling_logp_difference/mean": 1.7246109247207642,
      "step": 25,
      "step_time": 5.539267299005587
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 7.976094305515289,
      "epoch": 0.00026,
      "grad_norm": 0.030301710590720177,
      "kl": 0.13206800539046526,
      "learning_rate": 7.1428571428571436e-06,
      "loss": -0.0045,
      "step": 26,
      "step_time": 2.896310984997399
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 16.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.4375,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 7.269331395626068,
      "epoch": 0.00027,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.022369084879755974,
      "kl": 0.16119840927422047,
      "learning_rate": 7.428571428571429e-06,
      "loss": -0.0036,
      "num_tokens": 496650.0,
      "reward": -0.7243883013725281,
      "reward_std": 0.7688334584236145,
      "rewards/rollout_reward_func/mean": -0.7243883013725281,
      "rewards/rollout_reward_func/std": 0.7527879476547241,
      "sampling/importance_sampling_ratio/max": 0.18785437941551208,
      "sampling/importance_sampling_ratio/mean": 0.10117587447166443,
      "sampling/importance_sampling_ratio/min": 8.512477528421769e-11,
      "sampling/sampling_logp_difference/max": 4.909823417663574,
      "sampling/sampling_logp_difference/mean": 1.4340462684631348,
      "step": 27,
      "step_time": 6.4576256859945715
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 7.120794892311096,
      "epoch": 0.00028,
      "grad_norm": 0.02468658983707428,
      "kl": 0.182576522231102,
      "learning_rate": 7.714285714285716e-06,
      "loss": -0.0038,
      "step": 28,
      "step_time": 3.5662226489985187
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.0,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 7.656641006469727,
      "epoch": 0.00029,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.016346270218491554,
      "kl": 0.1884312927722931,
      "learning_rate": 8.000000000000001e-06,
      "loss": -0.0025,
      "num_tokens": 531329.0,
      "reward": -0.6714671850204468,
      "reward_std": 0.8514942526817322,
      "rewards/rollout_reward_func/mean": -0.6714671850204468,
      "rewards/rollout_reward_func/std": 0.8725821375846863,
      "sampling/importance_sampling_ratio/max": 0.2034609168767929,
      "sampling/importance_sampling_ratio/mean": 0.0898696631193161,
      "sampling/importance_sampling_ratio/min": 0.008383152075111866,
      "sampling/sampling_logp_difference/max": 2.7939882278442383,
      "sampling/sampling_logp_difference/mean": 1.5407953262329102,
      "step": 29,
      "step_time": 5.637962408003659
    },
    {
      "clip_ratio/high_max": 0.0625,
      "clip_ratio/high_mean": 0.046875,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.046875,
      "entropy": 7.5971901416778564,
      "epoch": 0.0003,
      "grad_norm": 0.013556623831391335,
      "kl": 0.20893656089901924,
      "learning_rate": 8.285714285714287e-06,
      "loss": -0.0026,
      "step": 30,
      "step_time": 2.9192343930008064
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9.0,
      "completions/max_terminated_length": 9.0,
      "completions/mean_length": 2.21875,
      "completions/mean_terminated_length": 2.21875,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 6.758796453475952,
      "epoch": 0.00031,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.019501112401485443,
      "kl": 0.3079346362501383,
      "learning_rate": 8.571428571428571e-06,
      "loss": -0.0031,
      "num_tokens": 567805.0,
      "reward": -0.568469762802124,
      "reward_std": 0.8567708730697632,
      "rewards/rollout_reward_func/mean": -0.568469762802124,
      "rewards/rollout_reward_func/std": 0.8660122752189636,
      "sampling/importance_sampling_ratio/max": 0.22180257737636566,
      "sampling/importance_sampling_ratio/mean": 0.12508273124694824,
      "sampling/importance_sampling_ratio/min": 3.750224089604792e-11,
      "sampling/sampling_logp_difference/max": 5.136954307556152,
      "sampling/sampling_logp_difference/mean": 1.4436562061309814,
      "step": 31,
      "step_time": 5.43906901200171
    },
    {
      "clip_ratio/high_max": 0.0625,
      "clip_ratio/high_mean": 0.03125,
      "clip_ratio/low_mean": 0.03125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0625,
      "entropy": 6.6603924036026,
      "epoch": 0.00032,
      "grad_norm": 0.01703478768467903,
      "kl": 0.3732527755200863,
      "learning_rate": 8.857142857142858e-06,
      "loss": -0.0032,
      "step": 32,
      "step_time": 2.930219074998604
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.0,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 6.725336015224457,
      "epoch": 0.00033,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.016738811507821083,
      "kl": 0.27567504718899727,
      "learning_rate": 9.142857142857144e-06,
      "loss": -0.0019,
      "num_tokens": 603268.0,
      "reward": -0.30585941672325134,
      "reward_std": 0.6699719429016113,
      "rewards/rollout_reward_func/mean": -0.30585941672325134,
      "rewards/rollout_reward_func/std": 0.6897762417793274,
      "sampling/importance_sampling_ratio/max": 0.2360040694475174,
      "sampling/importance_sampling_ratio/mean": 0.1374823898077011,
      "sampling/importance_sampling_ratio/min": 0.006810983642935753,
      "sampling/sampling_logp_difference/max": 3.1095614433288574,
      "sampling/sampling_logp_difference/mean": 1.2982618808746338,
      "step": 33,
      "step_time": 6.234361916005582
    },
    {
      "clip_ratio/high_max": 0.09375,
      "clip_ratio/high_mean": 0.046875,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.046875,
      "entropy": 6.653369903564453,
      "epoch": 0.00034,
      "grad_norm": 0.01500980369746685,
      "kl": 0.29385758377611637,
      "learning_rate": 9.42857142857143e-06,
      "loss": -0.0019,
      "step": 34,
      "step_time": 2.948668930999702
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5.0,
      "completions/max_terminated_length": 5.0,
      "completions/mean_length": 2.09375,
      "completions/mean_terminated_length": 2.09375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 6.599472224712372,
      "epoch": 0.00035,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.018205825239419937,
      "kl": 0.35661908239126205,
      "learning_rate": 9.714285714285715e-06,
      "loss": -0.0049,
      "num_tokens": 639208.0,
      "reward": -0.5350777506828308,
      "reward_std": 0.7106601595878601,
      "rewards/rollout_reward_func/mean": -0.5350777506828308,
      "rewards/rollout_reward_func/std": 0.7991757392883301,
      "sampling/importance_sampling_ratio/max": 0.25684407353401184,
      "sampling/importance_sampling_ratio/mean": 0.1485980749130249,
      "sampling/importance_sampling_ratio/min": 3.692734389915131e-05,
      "sampling/sampling_logp_difference/max": 4.381838321685791,
      "sampling/sampling_logp_difference/mean": 1.253082275390625,
      "step": 35,
      "step_time": 5.500268681997113
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 6.5274258852005005,
      "epoch": 0.00036,
      "grad_norm": 0.026637688279151917,
      "kl": 0.36279567517340183,
      "learning_rate": 1e-05,
      "loss": -0.0049,
      "step": 36,
      "step_time": 2.9163373929950467
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.0,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 6.327381074428558,
      "epoch": 0.00037,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.03791587054729462,
      "kl": 0.4374086819589138,
      "learning_rate": 9.999999999962232e-06,
      "loss": -0.0034,
      "num_tokens": 675224.0,
      "reward": -0.42839378118515015,
      "reward_std": 0.7165933847427368,
      "rewards/rollout_reward_func/mean": -0.42839378118515015,
      "rewards/rollout_reward_func/std": 0.6934623122215271,
      "sampling/importance_sampling_ratio/max": 0.2750149071216583,
      "sampling/importance_sampling_ratio/mean": 0.16812871396541595,
      "sampling/importance_sampling_ratio/min": 0.005278678145259619,
      "sampling/sampling_logp_difference/max": 3.2639646530151367,
      "sampling/sampling_logp_difference/mean": 1.122904896736145,
      "step": 37,
      "step_time": 5.68354233199716
    },
    {
      "clip_ratio/high_max": 0.09375,
      "clip_ratio/high_mean": 0.046875,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.046875,
      "entropy": 6.15696656703949,
      "epoch": 0.00038,
      "grad_norm": 0.01739896647632122,
      "kl": 0.46510135009884834,
      "learning_rate": 9.999999999848919e-06,
      "loss": -0.0035,
      "step": 38,
      "step_time": 2.9220271470039734
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 16.0,
      "completions/max_terminated_length": 8.0,
      "completions/mean_length": 2.625,
      "completions/mean_terminated_length": 2.1935482025146484,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 6.091022729873657,
      "epoch": 0.00039,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0197161715477705,
      "kl": 0.42657361552119255,
      "learning_rate": 9.99999999966007e-06,
      "loss": -0.0024,
      "num_tokens": 710988.0,
      "reward": -0.3023349940776825,
      "reward_std": 0.6465471386909485,
      "rewards/rollout_reward_func/mean": -0.3023349940776825,
      "rewards/rollout_reward_func/std": 0.6331813335418701,
      "sampling/importance_sampling_ratio/max": 0.2962448298931122,
      "sampling/importance_sampling_ratio/mean": 0.18444794416427612,
      "sampling/importance_sampling_ratio/min": 4.504835306867738e-12,
      "sampling/sampling_logp_difference/max": 4.963308334350586,
      "sampling/sampling_logp_difference/mean": 1.1856834888458252,
      "step": 39,
      "step_time": 7.029601453006762
    },
    {
      "clip_ratio/high_max": 0.1319444444961846,
      "clip_ratio/high_mean": 0.07847222150303423,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.07847222150303423,
      "entropy": 5.928341567516327,
      "epoch": 0.0004,
      "grad_norm": 0.028808562085032463,
      "kl": 0.44897962361574173,
      "learning_rate": 9.99999999939568e-06,
      "loss": -0.0025,
      "step": 40,
      "step_time": 2.9406937890053086
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.0,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 6.25755649805069,
      "epoch": 0.00041,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.0933663472533226,
      "kl": 0.48784746043384075,
      "learning_rate": 9.999999999055747e-06,
      "loss": 0.0029,
      "num_tokens": 745636.0,
      "reward": -0.19651329517364502,
      "reward_std": 0.5318358540534973,
      "rewards/rollout_reward_func/mean": -0.19651329517364502,
      "rewards/rollout_reward_func/std": 0.5945489406585693,
      "sampling/importance_sampling_ratio/max": 0.31440603733062744,
      "sampling/importance_sampling_ratio/mean": 0.18640094995498657,
      "sampling/importance_sampling_ratio/min": 0.011243580840528011,
      "sampling/sampling_logp_difference/max": 2.6481189727783203,
      "sampling/sampling_logp_difference/mean": 1.0152370929718018,
      "step": 41,
      "step_time": 5.63657486000011
    },
    {
      "clip_ratio/high_max": 0.1875,
      "clip_ratio/high_mean": 0.09375,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.09375,
      "entropy": 5.96447890996933,
      "epoch": 0.00042,
      "grad_norm": 0.02206423319876194,
      "kl": 0.5289704687893391,
      "learning_rate": 9.999999998640277e-06,
      "loss": 0.0027,
      "step": 42,
      "step_time": 2.8975234469944553
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.0,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 5.439069867134094,
      "epoch": 0.00043,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.11297624558210373,
      "kl": 0.4683471880853176,
      "learning_rate": 9.999999998149264e-06,
      "loss": 0.0006,
      "num_tokens": 781581.0,
      "reward": -0.44622302055358887,
      "reward_std": 0.68892902135849,
      "rewards/rollout_reward_func/mean": -0.44622302055358887,
      "rewards/rollout_reward_func/std": 0.7478122711181641,
      "sampling/importance_sampling_ratio/max": 0.3280465304851532,
      "sampling/importance_sampling_ratio/mean": 0.22684511542320251,
      "sampling/importance_sampling_ratio/min": 0.026074819266796112,
      "sampling/sampling_logp_difference/max": 2.0682249069213867,
      "sampling/sampling_logp_difference/mean": 0.8481977581977844,
      "step": 43,
      "step_time": 5.7263973810077005
    },
    {
      "clip_ratio/high_max": 0.15625,
      "clip_ratio/high_mean": 0.09375,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.09375,
      "entropy": 5.03247994184494,
      "epoch": 0.00044,
      "grad_norm": 0.06387817859649658,
      "kl": 0.5371211282908916,
      "learning_rate": 9.999999997582713e-06,
      "loss": 0.0004,
      "step": 44,
      "step_time": 3.3802060630041524
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5.0,
      "completions/max_terminated_length": 5.0,
      "completions/mean_length": 2.09375,
      "completions/mean_terminated_length": 2.09375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 4.975203037261963,
      "epoch": 0.00045,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.13102945685386658,
      "kl": 0.6513971909880638,
      "learning_rate": 9.999999996940621e-06,
      "loss": -0.0028,
      "num_tokens": 817273.0,
      "reward": -0.587563157081604,
      "reward_std": 0.7007678747177124,
      "rewards/rollout_reward_func/mean": -0.587563157081604,
      "rewards/rollout_reward_func/std": 0.7760494947433472,
      "sampling/importance_sampling_ratio/max": 0.3387902081012726,
      "sampling/importance_sampling_ratio/mean": 0.2464321404695511,
      "sampling/importance_sampling_ratio/min": 7.80636619310826e-05,
      "sampling/sampling_logp_difference/max": 4.357866287231445,
      "sampling/sampling_logp_difference/mean": 0.8462474346160889,
      "step": 45,
      "step_time": 6.2393272499975865
    },
    {
      "clip_ratio/high_max": 0.0625,
      "clip_ratio/high_mean": 0.03125,
      "clip_ratio/low_mean": 0.0625,
      "clip_ratio/low_min": 0.03125,
      "clip_ratio/region_mean": 0.09375,
      "entropy": 4.856449127197266,
      "epoch": 0.00046,
      "grad_norm": 0.0859452411532402,
      "kl": 0.6537227220833302,
      "learning_rate": 9.99999999622299e-06,
      "loss": -0.0031,
      "step": 46,
      "step_time": 2.901350881998951
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.0,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 4.492773771286011,
      "epoch": 0.00047,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.14956164360046387,
      "kl": 0.5590145848691463,
      "learning_rate": 9.999999995429816e-06,
      "loss": -0.0016,
      "num_tokens": 853016.0,
      "reward": -0.2771303355693817,
      "reward_std": 0.7537246942520142,
      "rewards/rollout_reward_func/mean": -0.2771303355693817,
      "rewards/rollout_reward_func/std": 0.7401061654090881,
      "sampling/importance_sampling_ratio/max": 0.34557926654815674,
      "sampling/importance_sampling_ratio/mean": 0.2762402594089508,
      "sampling/importance_sampling_ratio/min": 0.03973078727722168,
      "sampling/sampling_logp_difference/max": 1.9350109100341797,
      "sampling/sampling_logp_difference/mean": 0.7055625915527344,
      "step": 47,
      "step_time": 5.625663070004521
    },
    {
      "clip_ratio/high_max": 0.09375,
      "clip_ratio/high_mean": 0.046875,
      "clip_ratio/low_mean": 0.03125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.078125,
      "entropy": 4.3877677619457245,
      "epoch": 0.00048,
      "grad_norm": 0.06713134795427322,
      "kl": 0.578897014260292,
      "learning_rate": 9.999999994561102e-06,
      "loss": -0.0019,
      "step": 48,
      "step_time": 2.886007981996954
    },
    {
      "clip_ratio/high_max": 0.013888888992369175,
      "clip_ratio/high_mean": 0.0069444444961845875,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0069444444961845875,
      "completions/clipped_ratio": 0.0625,
      "completions/max_length": 16.0,
      "completions/max_terminated_length": 5.0,
      "completions/mean_length": 2.96875,
      "completions/mean_terminated_length": 2.1000001430511475,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 4.886862337589264,
      "epoch": 0.00049,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3026506304740906,
      "kl": 0.6998865567147732,
      "learning_rate": 9.99999999361685e-06,
      "loss": -0.0043,
      "num_tokens": 888637.0,
      "reward": -0.3646780252456665,
      "reward_std": 0.7392382025718689,
      "rewards/rollout_reward_func/mean": -0.3646780252456665,
      "rewards/rollout_reward_func/std": 0.7437232136726379,
      "sampling/importance_sampling_ratio/max": 0.5906126499176025,
      "sampling/importance_sampling_ratio/mean": 0.2536194622516632,
      "sampling/importance_sampling_ratio/min": 5.002554794020231e-12,
      "sampling/sampling_logp_difference/max": 5.405303001403809,
      "sampling/sampling_logp_difference/mean": 1.139528751373291,
      "step": 49,
      "step_time": 5.81962778799425
    },
    {
      "clip_ratio/high_max": 0.045138888992369175,
      "clip_ratio/high_mean": 0.03819444449618459,
      "clip_ratio/low_mean": 0.09375,
      "clip_ratio/low_min": 0.03125,
      "clip_ratio/region_mean": 0.13194444426335394,
      "entropy": 4.753438889980316,
      "epoch": 0.0005,
      "grad_norm": 0.22047115862369537,
      "kl": 0.8953660875558853,
      "learning_rate": 9.999999992597058e-06,
      "loss": -0.0044,
      "step": 50,
      "step_time": 3.4302545330028806
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.03125,
      "completions/max_length": 16.0,
      "completions/max_terminated_length": 11.0,
      "completions/mean_length": 2.71875,
      "completions/mean_terminated_length": 2.2903225421905518,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 4.553148508071899,
      "epoch": 0.00051,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.14803296327590942,
      "kl": 0.8594339191913605,
      "learning_rate": 9.999999991501723e-06,
      "loss": 0.0001,
      "num_tokens": 925783.0,
      "reward": -0.47938936948776245,
      "reward_std": 0.6224657893180847,
      "rewards/rollout_reward_func/mean": -0.47938936948776245,
      "rewards/rollout_reward_func/std": 0.6325410604476929,
      "sampling/importance_sampling_ratio/max": 0.8112522959709167,
      "sampling/importance_sampling_ratio/mean": 0.29810550808906555,
      "sampling/importance_sampling_ratio/min": 3.594766628464696e-13,
      "sampling/sampling_logp_difference/max": 5.109455108642578,
      "sampling/sampling_logp_difference/mean": 1.0870777368545532,
      "step": 51,
      "step_time": 6.738885986007517
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.02524038404226303,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.02524038404226303,
      "entropy": 4.535911321640015,
      "epoch": 0.00052,
      "grad_norm": 0.04543463885784149,
      "kl": 0.7985228635370731,
      "learning_rate": 9.99999999033085e-06,
      "loss": -0.0004,
      "step": 52,
      "step_time": 3.152213782999752
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2.0,
      "completions/max_terminated_length": 2.0,
      "completions/mean_length": 2.0,
      "completions/mean_terminated_length": 2.0,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 4.083370506763458,
      "epoch": 0.00053,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.04525892809033394,
      "kl": 0.874318428337574,
      "learning_rate": 9.999999989084436e-06,
      "loss": -0.0025,
      "num_tokens": 961105.0,
      "reward": -0.21413108706474304,
      "reward_std": 0.5813945531845093,
      "rewards/rollout_reward_func/mean": -0.21413108706474304,
      "rewards/rollout_reward_func/std": 0.5861169099807739,
      "sampling/importance_sampling_ratio/max": 0.7165222764015198,
      "sampling/importance_sampling_ratio/mean": 0.3133776783943176,
      "sampling/importance_sampling_ratio/min": 0.013980884104967117,
      "sampling/sampling_logp_difference/max": 3.070335626602173,
      "sampling/sampling_logp_difference/mean": 0.7299262285232544,
      "step": 53,
      "step_time": 6.067928731994471
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.015625,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.03125,
      "entropy": 4.059947609901428,
      "epoch": 0.00054,
      "grad_norm": 0.07141973823308945,
      "kl": 0.9976279065012932,
      "learning_rate": 9.99999998776248e-06,
      "loss": -0.0025,
      "step": 54,
      "step_time": 3.118995607001125
    },
    {
      "clip_ratio/high_max": 0.03125,
      "clip_ratio/high_mean": 0.015625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 5.0,
      "completions/max_terminated_length": 5.0,
      "completions/mean_length": 2.09375,
      "completions/mean_terminated_length": 2.09375,
      "completions/min_length": 2.0,
      "completions/min_terminated_length": 2.0,
      "entropy": 4.261334180831909,
      "epoch": 0.00055,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06153355911374092,
      "kl": 0.8010737895965576,
      "learning_rate": 9.999999986364988e-06,
      "loss": 0.0033,
      "num_tokens": 996705.0,
      "reward": -0.33633241057395935,
      "reward_std": 0.4821242392063141,
      "rewards/rollout_reward_func/mean": -0.33633241057395935,
      "rewards/rollout_reward_func/std": 0.5220240354537964,
      "sampling/importance_sampling_ratio/max": 0.9735277891159058,
      "sampling/importance_sampling_ratio/mean": 0.2968878149986267,
      "sampling/importance_sampling_ratio/min": 0.0002361015067435801,
      "sampling/sampling_logp_difference/max": 4.416370868682861,
      "sampling/sampling_logp_difference/mean": 0.8390293717384338,
      "step": 55,
      "step_time": 5.6938710109971
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 700000,
  "num_input_tokens_seen": 996705,
  "num_train_epochs": 7,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}