{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.006,
  "eval_steps": 500,
  "global_step": 75,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2813.0,
      "completions/max_terminated_length": 2813.0,
      "completions/mean_length": 2062.46875,
      "completions/mean_terminated_length": 2062.46875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.1340037016198039,
      "epoch": 8e-05,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 0.9753277897834778,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": 0.0097,
      "num_tokens": 78941.0,
      "reward": 0.5106250047683716,
      "reward_std": 0.16591878235340118,
      "rewards/rollout_reward_func/mean": 0.5106250047683716,
      "rewards/rollout_reward_func/std": 0.38574549555778503,
      "sampling/importance_sampling_ratio/max": 1.89468514919281,
      "sampling/importance_sampling_ratio/mean": 0.917938768863678,
      "sampling/importance_sampling_ratio/min": 0.26035696268081665,
      "sampling/sampling_logp_difference/max": 1.035329818725586,
      "sampling/sampling_logp_difference/mean": 0.020964600145816803,
      "step": 1,
      "step_time": 18.408817325000086
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2795.0,
      "completions/max_terminated_length": 2795.0,
      "completions/mean_length": 2091.09375,
      "completions/mean_terminated_length": 2091.09375,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.1304742144420743,
      "epoch": 0.00016,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.355797290802002,
      "kl": 0.0,
      "learning_rate": 2.2857142857142855e-07,
      "loss": -0.0694,
      "num_tokens": 158774.0,
      "reward": 0.38593748211860657,
      "reward_std": 0.15246255695819855,
      "rewards/rollout_reward_func/mean": 0.38593748211860657,
      "rewards/rollout_reward_func/std": 0.3016391694545746,
      "sampling/importance_sampling_ratio/max": 2.747450828552246,
      "sampling/importance_sampling_ratio/mean": 0.995655357837677,
      "sampling/importance_sampling_ratio/min": 0.30046260356903076,
      "sampling/sampling_logp_difference/max": 1.1795392036437988,
      "sampling/sampling_logp_difference/mean": 0.022426610812544823,
      "step": 2,
      "step_time": 17.007036188999905
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2783.0,
      "completions/max_terminated_length": 2783.0,
      "completions/mean_length": 1881.75,
      "completions/mean_terminated_length": 1881.75,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.12374210823327303,
      "epoch": 0.00024,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 0.6671572327613831,
      "kl": 0.0014705628045703634,
      "learning_rate": 4.571428571428571e-07,
      "loss": -0.0235,
      "num_tokens": 231260.0,
      "reward": 0.4012500047683716,
      "reward_std": 0.20683754980564117,
      "rewards/rollout_reward_func/mean": 0.4012500047683716,
      "rewards/rollout_reward_func/std": 0.33187392354011536,
      "sampling/importance_sampling_ratio/max": 1.3709925413131714,
      "sampling/importance_sampling_ratio/mean": 0.8955257534980774,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9760880470275879,
      "sampling/sampling_logp_difference/mean": 0.020091338083148003,
      "step": 3,
      "step_time": 16.527492177000227
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2795.0,
      "completions/max_terminated_length": 2795.0,
      "completions/mean_length": 2263.03125,
      "completions/mean_terminated_length": 2263.03125,
      "completions/min_length": 1569.0,
      "completions/min_terminated_length": 1569.0,
      "entropy": 0.15253359219059348,
      "epoch": 0.00032,
      "frac_reward_zero_std": 0.875,
      "grad_norm": 0.9312232136726379,
      "kl": 0.0016055026353569701,
      "learning_rate": 6.857142857142857e-07,
      "loss": -0.0241,
      "num_tokens": 316879.0,
      "reward": 0.3787500262260437,
      "reward_std": 0.0624999962747097,
      "rewards/rollout_reward_func/mean": 0.3787500262260437,
      "rewards/rollout_reward_func/std": 0.26268768310546875,
      "sampling/importance_sampling_ratio/max": 1.9853609800338745,
      "sampling/importance_sampling_ratio/mean": 0.9613277912139893,
      "sampling/importance_sampling_ratio/min": 0.4037262201309204,
      "sampling/sampling_logp_difference/max": 0.6126779317855835,
      "sampling/sampling_logp_difference/mean": 0.02291969209909439,
      "step": 4,
      "step_time": 18.10109623099993
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.003289473708719015,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003289473708719015,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2807.0,
      "completions/max_terminated_length": 2807.0,
      "completions/mean_length": 2197.1875,
      "completions/mean_terminated_length": 2197.1875,
      "completions/min_length": 1570.0,
      "completions/min_terminated_length": 1570.0,
      "entropy": 0.14702600054442883,
      "epoch": 0.0004,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 2.2064177989959717,
      "kl": 0.002026251120696543,
      "learning_rate": 9.142857142857142e-07,
      "loss": -0.0169,
      "num_tokens": 400370.0,
      "reward": 0.4140625,
      "reward_std": 0.15217570960521698,
      "rewards/rollout_reward_func/mean": 0.4140625,
      "rewards/rollout_reward_func/std": 0.33838188648223877,
      "sampling/importance_sampling_ratio/max": 2.348391532897949,
      "sampling/importance_sampling_ratio/mean": 1.0998704433441162,
      "sampling/importance_sampling_ratio/min": 0.6395935416221619,
      "sampling/sampling_logp_difference/max": 0.6357507705688477,
      "sampling/sampling_logp_difference/mean": 0.020555175840854645,
      "step": 5,
      "step_time": 16.967482953000058
    },
    {
      "clip_ratio/high_max": 0.009868421126157045,
      "clip_ratio/high_mean": 0.004934210563078523,
      "clip_ratio/low_mean": 0.0034829721553251147,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008417182718403637,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2449.0,
      "completions/max_terminated_length": 2449.0,
      "completions/mean_length": 1818.28125,
      "completions/mean_terminated_length": 1818.28125,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.0911772302351892,
      "epoch": 0.00048,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 0.8283352255821228,
      "kl": 0.001795254382159328,
      "learning_rate": 1.1428571428571428e-06,
      "loss": -0.0209,
      "num_tokens": 470216.0,
      "reward": 0.5893750190734863,
      "reward_std": 0.1562499850988388,
      "rewards/rollout_reward_func/mean": 0.5893750190734863,
      "rewards/rollout_reward_func/std": 0.4223737418651581,
      "sampling/importance_sampling_ratio/max": 1.7877978086471558,
      "sampling/importance_sampling_ratio/mean": 0.9894595146179199,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9945626258850098,
      "sampling/sampling_logp_difference/mean": 0.01755390875041485,
      "step": 6,
      "step_time": 15.150656265999714
    },
    {
      "clip_ratio/high_max": 0.009375000139698386,
      "clip_ratio/high_mean": 0.0062500000931322575,
      "clip_ratio/low_mean": 0.006411405862309039,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.012661405955441296,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2791.0,
      "completions/max_terminated_length": 2791.0,
      "completions/mean_length": 2104.28125,
      "completions/mean_terminated_length": 2104.28125,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.16090343240648508,
      "epoch": 0.00056,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.4131731986999512,
      "kl": 0.002872211887734011,
      "learning_rate": 1.3714285714285715e-06,
      "loss": -0.03,
      "num_tokens": 550373.0,
      "reward": 0.29500001668930054,
      "reward_std": 0.0949999988079071,
      "rewards/rollout_reward_func/mean": 0.29500001668930054,
      "rewards/rollout_reward_func/std": 0.1687716543674469,
      "sampling/importance_sampling_ratio/max": 1.3797976970672607,
      "sampling/importance_sampling_ratio/mean": 0.9415616989135742,
      "sampling/importance_sampling_ratio/min": 0.29769256711006165,
      "sampling/sampling_logp_difference/max": 0.9464168548583984,
      "sampling/sampling_logp_difference/mean": 0.02406277321279049,
      "step": 7,
      "step_time": 16.95326116199999
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0016447368543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0016447368543595076,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2814.0,
      "completions/max_terminated_length": 2814.0,
      "completions/mean_length": 2265.5625,
      "completions/mean_terminated_length": 2265.5625,
      "completions/min_length": 1569.0,
      "completions/min_terminated_length": 1569.0,
      "entropy": 0.20542557537555695,
      "epoch": 0.00064,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.0486057996749878,
      "kl": 0.00249500987411011,
      "learning_rate": 1.6e-06,
      "loss": 0.0307,
      "num_tokens": 636318.0,
      "reward": 0.3434374928474426,
      "reward_std": 0.08029377460479736,
      "rewards/rollout_reward_func/mean": 0.3434374928474426,
      "rewards/rollout_reward_func/std": 0.22245851159095764,
      "sampling/importance_sampling_ratio/max": 2.239882230758667,
      "sampling/importance_sampling_ratio/mean": 0.9191794395446777,
      "sampling/importance_sampling_ratio/min": 0.3405879735946655,
      "sampling/sampling_logp_difference/max": 1.0898922681808472,
      "sampling/sampling_logp_difference/mean": 0.023122236132621765,
      "step": 8,
      "step_time": 18.007336240000086
    },
    {
      "clip_ratio/high_max": 0.010620915098115802,
      "clip_ratio/high_mean": 0.005310457549057901,
      "clip_ratio/low_mean": 0.0016447368543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0069551944034174085,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2790.0,
      "completions/max_terminated_length": 2790.0,
      "completions/mean_length": 1646.03125,
      "completions/mean_terminated_length": 1646.03125,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.13724102126434445,
      "epoch": 0.00072,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 1.481195092201233,
      "kl": 0.0015346993204730097,
      "learning_rate": 1.8285714285714284e-06,
      "loss": -0.0189,
      "num_tokens": 701005.0,
      "reward": 0.6278125047683716,
      "reward_std": 0.30871257185935974,
      "rewards/rollout_reward_func/mean": 0.6278125047683716,
      "rewards/rollout_reward_func/std": 0.4518972933292389,
      "sampling/importance_sampling_ratio/max": 1.9988352060317993,
      "sampling/importance_sampling_ratio/mean": 0.9468162059783936,
      "sampling/importance_sampling_ratio/min": 0.4476884603500366,
      "sampling/sampling_logp_difference/max": 0.773470401763916,
      "sampling/sampling_logp_difference/mean": 0.02107076346874237,
      "step": 9,
      "step_time": 16.103736942000182
    },
    {
      "clip_ratio/high_max": 0.008938953513279557,
      "clip_ratio/high_mean": 0.004469476756639779,
      "clip_ratio/low_mean": 0.0015243901871144772,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005993866943754256,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2799.0,
      "completions/max_terminated_length": 2799.0,
      "completions/mean_length": 2226.4375,
      "completions/mean_terminated_length": 2226.4375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.18680323101580143,
      "epoch": 0.0008,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 2.0065183639526367,
      "kl": 0.002535051797167398,
      "learning_rate": 2.057142857142857e-06,
      "loss": 0.0495,
      "num_tokens": 785722.0,
      "reward": 0.4443749785423279,
      "reward_std": 0.08841878175735474,
      "rewards/rollout_reward_func/mean": 0.4443749785423279,
      "rewards/rollout_reward_func/std": 0.35590803623199463,
      "sampling/importance_sampling_ratio/max": 2.790905714035034,
      "sampling/importance_sampling_ratio/mean": 0.940368115901947,
      "sampling/importance_sampling_ratio/min": 0.22617992758750916,
      "sampling/sampling_logp_difference/max": 0.6851506233215332,
      "sampling/sampling_logp_difference/mean": 0.025203729048371315,
      "step": 10,
      "step_time": 17.030096009999966
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0015625000232830644,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0015625000232830644,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2783.0,
      "completions/max_terminated_length": 2783.0,
      "completions/mean_length": 1966.875,
      "completions/mean_terminated_length": 1966.875,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.1440325272269547,
      "epoch": 0.00088,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.0474733114242554,
      "kl": 0.0011928395033464767,
      "learning_rate": 2.2857142857142856e-06,
      "loss": -0.0121,
      "num_tokens": 861366.0,
      "reward": 0.4596875011920929,
      "reward_std": 0.14279377460479736,
      "rewards/rollout_reward_func/mean": 0.4596875011920929,
      "rewards/rollout_reward_func/std": 0.38282889127731323,
      "sampling/importance_sampling_ratio/max": 1.9087510108947754,
      "sampling/importance_sampling_ratio/mean": 0.9341506361961365,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.9133691787719727,
      "sampling/sampling_logp_difference/mean": 0.02222413383424282,
      "step": 11,
      "step_time": 16.685985845999994
    },
    {
      "clip_ratio/high_max": 0.007352941203862429,
      "clip_ratio/high_mean": 0.0055147059028968215,
      "clip_ratio/low_mean": 0.0036764706019312143,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009191176504828036,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2801.0,
      "completions/max_terminated_length": 2801.0,
      "completions/mean_length": 1857.96875,
      "completions/mean_terminated_length": 1857.96875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.17092999629676342,
      "epoch": 0.00096,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 2.0095906257629395,
      "kl": 0.0024750066513661295,
      "learning_rate": 2.5142857142857142e-06,
      "loss": -0.0228,
      "num_tokens": 933262.0,
      "reward": 0.48593753576278687,
      "reward_std": 0.20529377460479736,
      "rewards/rollout_reward_func/mean": 0.48593753576278687,
      "rewards/rollout_reward_func/std": 0.38762184977531433,
      "sampling/importance_sampling_ratio/max": 1.6060364246368408,
      "sampling/importance_sampling_ratio/mean": 0.9451028108596802,
      "sampling/importance_sampling_ratio/min": 0.34411877393722534,
      "sampling/sampling_logp_difference/max": 1.0912601947784424,
      "sampling/sampling_logp_difference/mean": 0.019420120865106583,
      "step": 12,
      "step_time": 17.06754343600005
    },
    {
      "clip_ratio/high_max": 0.007936508161947131,
      "clip_ratio/high_mean": 0.003968254080973566,
      "clip_ratio/low_mean": 0.0017361111240461469,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0057043652050197124,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2812.0,
      "completions/max_terminated_length": 2812.0,
      "completions/mean_length": 1741.5,
      "completions/mean_terminated_length": 1741.5,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.11786343855783343,
      "epoch": 0.00104,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 2.0365312099456787,
      "kl": 0.0034534272163000423,
      "learning_rate": 2.742857142857143e-06,
      "loss": -0.0389,
      "num_tokens": 1001046.0,
      "reward": 0.6475000381469727,
      "reward_std": 0.24292194843292236,
      "rewards/rollout_reward_func/mean": 0.6475000381469727,
      "rewards/rollout_reward_func/std": 0.4391413629055023,
      "sampling/importance_sampling_ratio/max": 2.502153158187866,
      "sampling/importance_sampling_ratio/mean": 1.0042061805725098,
      "sampling/importance_sampling_ratio/min": 0.4875127375125885,
      "sampling/sampling_logp_difference/max": 0.6685242652893066,
      "sampling/sampling_logp_difference/mean": 0.01504062581807375,
      "step": 13,
      "step_time": 16.125135786999863
    },
    {
      "clip_ratio/high_max": 0.011430230224505067,
      "clip_ratio/high_mean": 0.0057151151122525334,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007947258069179952,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2802.0,
      "completions/max_terminated_length": 2802.0,
      "completions/mean_length": 2010.3125,
      "completions/mean_terminated_length": 2010.3125,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.1693209670484066,
      "epoch": 0.00112,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.2513231039047241,
      "kl": 0.002481764371623285,
      "learning_rate": 2.9714285714285716e-06,
      "loss": 0.0309,
      "num_tokens": 1078101.0,
      "reward": 0.4753125011920929,
      "reward_std": 0.19296419620513916,
      "rewards/rollout_reward_func/mean": 0.4753125011920929,
      "rewards/rollout_reward_func/std": 0.3761346936225891,
      "sampling/importance_sampling_ratio/max": 1.5099362134933472,
      "sampling/importance_sampling_ratio/mean": 0.9022589921951294,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.6567137241363525,
      "sampling/sampling_logp_difference/mean": 0.022564683109521866,
      "step": 14,
      "step_time": 16.688260055
    },
    {
      "clip_ratio/high_max": 0.0036764706019312143,
      "clip_ratio/high_mean": 0.0018382353009656072,
      "clip_ratio/low_mean": 0.0017361111240461469,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003574346425011754,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2775.0,
      "completions/max_terminated_length": 2775.0,
      "completions/mean_length": 1887.71875,
      "completions/mean_terminated_length": 1887.71875,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.14878523536026478,
      "epoch": 0.0012,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.092544674873352,
      "kl": 0.0021166762671782635,
      "learning_rate": 3.2e-06,
      "loss": -0.014,
      "num_tokens": 1150803.0,
      "reward": 0.4506249725818634,
      "reward_std": 0.2620203495025635,
      "rewards/rollout_reward_func/mean": 0.4506249725818634,
      "rewards/rollout_reward_func/std": 0.3878471255302429,
      "sampling/importance_sampling_ratio/max": 2.092060089111328,
      "sampling/importance_sampling_ratio/mean": 1.0061091184616089,
      "sampling/importance_sampling_ratio/min": 0.519212543964386,
      "sampling/sampling_logp_difference/max": 0.664109468460083,
      "sampling/sampling_logp_difference/mean": 0.02180980145931244,
      "step": 15,
      "step_time": 16.993085070999996
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0016891892300918698,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0016891892300918698,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2812.0,
      "completions/max_terminated_length": 2812.0,
      "completions/mean_length": 2229.28125,
      "completions/mean_terminated_length": 2229.28125,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.15634657256305218,
      "epoch": 0.00128,
      "frac_reward_zero_std": 0.875,
      "grad_norm": 0.09010659158229828,
      "kl": 0.005270412558274984,
      "learning_rate": 3.428571428571428e-06,
      "loss": 0.005,
      "num_tokens": 1235464.0,
      "reward": 0.48374998569488525,
      "reward_std": 0.0625,
      "rewards/rollout_reward_func/mean": 0.48374998569488525,
      "rewards/rollout_reward_func/std": 0.36630454659461975,
      "sampling/importance_sampling_ratio/max": 1.890110969543457,
      "sampling/importance_sampling_ratio/mean": 0.9257134199142456,
      "sampling/importance_sampling_ratio/min": 0.4630853831768036,
      "sampling/sampling_logp_difference/max": 0.69629967212677,
      "sampling/sampling_logp_difference/mean": 0.020435180515050888,
      "step": 16,
      "step_time": 17.667978367999467
    },
    {
      "clip_ratio/high_max": 0.016176471021026373,
      "clip_ratio/high_mean": 0.009732972481288016,
      "clip_ratio/low_mean": 0.0016447368543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011377709335647523,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2449.0,
      "completions/max_terminated_length": 2449.0,
      "completions/mean_length": 1750.53125,
      "completions/mean_terminated_length": 1750.53125,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.13329231040552258,
      "epoch": 0.00136,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.2919148206710815,
      "kl": 0.0019803230388788506,
      "learning_rate": 3.657142857142857e-06,
      "loss": 0.0395,
      "num_tokens": 1303537.0,
      "reward": 0.5606250166893005,
      "reward_std": 0.2351399064064026,
      "rewards/rollout_reward_func/mean": 0.5606250166893005,
      "rewards/rollout_reward_func/std": 0.41600972414016724,
      "sampling/importance_sampling_ratio/max": 1.4119406938552856,
      "sampling/importance_sampling_ratio/mean": 0.9287126064300537,
      "sampling/importance_sampling_ratio/min": 0.3827318847179413,
      "sampling/sampling_logp_difference/max": 0.6636209487915039,
      "sampling/sampling_logp_difference/mean": 0.017019610852003098,
      "step": 17,
      "step_time": 15.075951100999873
    },
    {
      "clip_ratio/high_max": 0.0062500000931322575,
      "clip_ratio/high_mean": 0.0031250000465661287,
      "clip_ratio/low_mean": 0.0016447368543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004769736900925636,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2799.0,
      "completions/max_terminated_length": 2799.0,
      "completions/mean_length": 2316.9375,
      "completions/mean_terminated_length": 2316.9375,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.16856362204998732,
      "epoch": 0.00144,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 2.867643356323242,
      "kl": 0.005084036383777857,
      "learning_rate": 3.885714285714286e-06,
      "loss": -0.0049,
      "num_tokens": 1391780.0,
      "reward": 0.37031251192092896,
      "reward_std": 0.10187499970197678,
      "rewards/rollout_reward_func/mean": 0.37031251192092896,
      "rewards/rollout_reward_func/std": 0.2657124996185303,
      "sampling/importance_sampling_ratio/max": 1.9782981872558594,
      "sampling/importance_sampling_ratio/mean": 1.0121815204620361,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.48717403411865234,
      "sampling/sampling_logp_difference/mean": 0.022180214524269104,
      "step": 18,
      "step_time": 17.064124244999903
    },
    {
      "clip_ratio/high_max": 0.0062500000931322575,
      "clip_ratio/high_mean": 0.0031250000465661287,
      "clip_ratio/low_mean": 0.0015243901871144772,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004649390233680606,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2798.0,
      "completions/max_terminated_length": 2798.0,
      "completions/mean_length": 2216.0,
      "completions/mean_terminated_length": 2216.0,
      "completions/min_length": 1563.0,
      "completions/min_terminated_length": 1563.0,
      "entropy": 0.1883529694750905,
      "epoch": 0.00152,
      "frac_reward_zero_std": 0.875,
      "grad_norm": 2.2749135494232178,
      "kl": 0.005091317143524066,
      "learning_rate": 4.114285714285714e-06,
      "loss": 0.0455,
      "num_tokens": 1475873.0,
      "reward": 0.4059374928474426,
      "reward_std": 0.00812500063329935,
      "rewards/rollout_reward_func/mean": 0.4059374928474426,
      "rewards/rollout_reward_func/std": 0.29813244938850403,
      "sampling/importance_sampling_ratio/max": 2.81974196434021,
      "sampling/importance_sampling_ratio/mean": 1.0345871448516846,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.8696746826171875,
      "sampling/sampling_logp_difference/mean": 0.02671925723552704,
      "step": 19,
      "step_time": 17.602816416999985
    },
    {
      "clip_ratio/high_max": 0.011488970601931214,
      "clip_ratio/high_mean": 0.005744485300965607,
      "clip_ratio/low_mean": 0.0048926768358796835,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.01063716213684529,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2819.0,
      "completions/max_terminated_length": 2819.0,
      "completions/mean_length": 1814.28125,
      "completions/mean_terminated_length": 1814.28125,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.1282729902304709,
      "epoch": 0.0016,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.691245198249817,
      "kl": 0.002560590350185521,
      "learning_rate": 4.342857142857142e-06,
      "loss": -0.0259,
      "num_tokens": 1546081.0,
      "reward": 0.612500011920929,
      "reward_std": 0.18216876685619354,
      "rewards/rollout_reward_func/mean": 0.612500011920929,
      "rewards/rollout_reward_func/std": 0.43820008635520935,
      "sampling/importance_sampling_ratio/max": 1.9052116870880127,
      "sampling/importance_sampling_ratio/mean": 1.007737636566162,
      "sampling/importance_sampling_ratio/min": 0.5377175211906433,
      "sampling/sampling_logp_difference/max": 0.6581223011016846,
      "sampling/sampling_logp_difference/mean": 0.01716558076441288,
      "step": 20,
      "step_time": 16.950590521000322
    },
    {
      "clip_ratio/high_max": 0.010667945956811309,
      "clip_ratio/high_mean": 0.005333972978405654,
      "clip_ratio/low_mean": 0.004784891498275101,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010118864476680756,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2791.0,
      "completions/max_terminated_length": 2791.0,
      "completions/mean_length": 1928.25,
      "completions/mean_terminated_length": 1928.25,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.1822828585281968,
      "epoch": 0.00168,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.4209487438201904,
      "kl": 0.0024251511349575594,
      "learning_rate": 4.571428571428571e-06,
      "loss": -0.038,
      "num_tokens": 1620395.0,
      "reward": 0.6059374809265137,
      "reward_std": 0.20529377460479736,
      "rewards/rollout_reward_func/mean": 0.6059374809265137,
      "rewards/rollout_reward_func/std": 0.4463822841644287,
      "sampling/importance_sampling_ratio/max": 1.6781816482543945,
      "sampling/importance_sampling_ratio/mean": 1.0399606227874756,
      "sampling/importance_sampling_ratio/min": 0.30244705080986023,
      "sampling/sampling_logp_difference/max": 0.7038769721984863,
      "sampling/sampling_logp_difference/mean": 0.028661729767918587,
      "step": 21,
      "step_time": 16.793115096000065
    },
    {
      "clip_ratio/high_max": 0.01126575656235218,
      "clip_ratio/high_mean": 0.00563287828117609,
      "clip_ratio/low_mean": 0.0018382353009656072,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007471113582141697,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2806.0,
      "completions/max_terminated_length": 2806.0,
      "completions/mean_length": 1895.0,
      "completions/mean_terminated_length": 1895.0,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.11727871629409492,
      "epoch": 0.00176,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.4466348886489868,
      "kl": 0.003775272169150412,
      "learning_rate": 4.8e-06,
      "loss": 0.0009,
      "num_tokens": 1693497.0,
      "reward": 0.5303125381469727,
      "reward_std": 0.2740437984466553,
      "rewards/rollout_reward_func/mean": 0.5303125381469727,
      "rewards/rollout_reward_func/std": 0.4203799068927765,
      "sampling/importance_sampling_ratio/max": 1.6391761302947998,
      "sampling/importance_sampling_ratio/mean": 0.9023667573928833,
      "sampling/importance_sampling_ratio/min": 0.30269956588745117,
      "sampling/sampling_logp_difference/max": 1.103229284286499,
      "sampling/sampling_logp_difference/mean": 0.028024764731526375,
      "step": 22,
      "step_time": 16.31928637299984
    },
    {
      "clip_ratio/high_max": 0.006761695956811309,
      "clip_ratio/high_mean": 0.0033808479784056544,
      "clip_ratio/low_mean": 0.0034829722717404366,
      "clip_ratio/low_min": 0.003289473708719015,
      "clip_ratio/region_mean": 0.006863820250146091,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2795.0,
      "completions/max_terminated_length": 2795.0,
      "completions/mean_length": 1923.9375,
      "completions/mean_terminated_length": 1923.9375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.14631808176636696,
      "epoch": 0.00184,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.9489635229110718,
      "kl": 0.0022731795979780145,
      "learning_rate": 5.0285714285714285e-06,
      "loss": -0.0229,
      "num_tokens": 1767354.0,
      "reward": 0.4168750047683716,
      "reward_std": 0.20417675375938416,
      "rewards/rollout_reward_func/mean": 0.4168750047683716,
      "rewards/rollout_reward_func/std": 0.33288994431495667,
      "sampling/importance_sampling_ratio/max": 2.5520823001861572,
      "sampling/importance_sampling_ratio/mean": 1.124953269958496,
      "sampling/importance_sampling_ratio/min": 0.3814745247364044,
      "sampling/sampling_logp_difference/max": 0.9945569038391113,
      "sampling/sampling_logp_difference/mean": 0.021298212930560112,
      "step": 23,
      "step_time": 16.909164390999877
    },
    {
      "clip_ratio/high_max": 0.0062500000931322575,
      "clip_ratio/high_mean": 0.0031250000465661287,
      "clip_ratio/low_mean": 0.0015625000232830644,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004687500069849193,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2798.0,
      "completions/max_terminated_length": 2798.0,
      "completions/mean_length": 2009.28125,
      "completions/mean_terminated_length": 2009.28125,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.18214968033134937,
      "epoch": 0.00192,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.610776662826538,
      "kl": 0.003005845101142768,
      "learning_rate": 5.257142857142857e-06,
      "loss": -0.0704,
      "num_tokens": 1844719.0,
      "reward": 0.4243749976158142,
      "reward_std": 0.21341876685619354,
      "rewards/rollout_reward_func/mean": 0.4243749976158142,
      "rewards/rollout_reward_func/std": 0.362561970949173,
      "sampling/importance_sampling_ratio/max": 1.7062076330184937,
      "sampling/importance_sampling_ratio/mean": 1.0438251495361328,
      "sampling/importance_sampling_ratio/min": 0.3492918312549591,
      "sampling/sampling_logp_difference/max": 0.5720778703689575,
      "sampling/sampling_logp_difference/mean": 0.023630155250430107,
      "step": 24,
      "step_time": 17.14499585999988
    },
    {
      "clip_ratio/high_max": 0.0031250000465661287,
      "clip_ratio/high_mean": 0.0015625000232830644,
      "clip_ratio/low_mean": 0.0031250000465661287,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004687500069849193,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2440.0,
      "completions/max_terminated_length": 2440.0,
      "completions/mean_length": 2006.6875,
      "completions/mean_terminated_length": 2006.6875,
      "completions/min_length": 1055.0,
      "completions/min_terminated_length": 1055.0,
      "entropy": 0.11316484399139881,
      "epoch": 0.002,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.0315803289413452,
      "kl": 0.0030158030276652426,
      "learning_rate": 5.485714285714286e-06,
      "loss": 0.0116,
      "num_tokens": 1921270.0,
      "reward": 0.47968751192092896,
      "reward_std": 0.07062499970197678,
      "rewards/rollout_reward_func/mean": 0.47968751192092896,
      "rewards/rollout_reward_func/std": 0.36560583114624023,
      "sampling/importance_sampling_ratio/max": 1.54865562915802,
      "sampling/importance_sampling_ratio/mean": 0.9767700433731079,
      "sampling/importance_sampling_ratio/min": 0.5038349628448486,
      "sampling/sampling_logp_difference/max": 0.4957547187805176,
      "sampling/sampling_logp_difference/mean": 0.01319466158747673,
      "step": 25,
      "step_time": 15.574967514000036
    },
    {
      "clip_ratio/high_max": 0.007352941203862429,
      "clip_ratio/high_mean": 0.0036764706019312143,
      "clip_ratio/low_mean": 0.0015625000232830644,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005238970625214279,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2816.0,
      "completions/max_terminated_length": 2816.0,
      "completions/mean_length": 2016.375,
      "completions/mean_terminated_length": 2016.375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.1327181551605463,
      "epoch": 0.00208,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.1481389999389648,
      "kl": 0.0036689931839646306,
      "learning_rate": 5.7142857142857145e-06,
      "loss": -0.004,
      "num_tokens": 1998617.0,
      "reward": 0.4909375011920929,
      "reward_std": 0.12780338525772095,
      "rewards/rollout_reward_func/mean": 0.4909375011920929,
      "rewards/rollout_reward_func/std": 0.3668818771839142,
      "sampling/importance_sampling_ratio/max": 1.5999431610107422,
      "sampling/importance_sampling_ratio/mean": 0.9634629487991333,
      "sampling/importance_sampling_ratio/min": 0.2564904987812042,
      "sampling/sampling_logp_difference/max": 0.6982665061950684,
      "sampling/sampling_logp_difference/mean": 0.01972239464521408,
      "step": 26,
      "step_time": 17.251899020999645
    },
    {
      "clip_ratio/high_max": 0.008928571827709675,
      "clip_ratio/high_mean": 0.004464285913854837,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004464285913854837,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2810.0,
      "completions/max_terminated_length": 2810.0,
      "completions/mean_length": 1849.84375,
      "completions/mean_terminated_length": 1849.84375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.09636542806401849,
      "epoch": 0.00216,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.7088967561721802,
      "kl": 0.002837517200532602,
      "learning_rate": 5.942857142857143e-06,
      "loss": -0.0039,
      "num_tokens": 2070057.0,
      "reward": 0.5687500238418579,
      "reward_std": 0.2160891890525818,
      "rewards/rollout_reward_func/mean": 0.5687500238418579,
      "rewards/rollout_reward_func/std": 0.4140106439590454,
      "sampling/importance_sampling_ratio/max": 1.6554824113845825,
      "sampling/importance_sampling_ratio/mean": 1.0057581663131714,
      "sampling/importance_sampling_ratio/min": 0.13771295547485352,
      "sampling/sampling_logp_difference/max": 1.693850040435791,
      "sampling/sampling_logp_difference/mean": 0.015657048672437668,
      "step": 27,
      "step_time": 16.657898435000106
    },
    {
      "clip_ratio/high_max": 0.0036764706019312143,
      "clip_ratio/high_mean": 0.0018382353009656072,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0018382353009656072,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2800.0,
      "completions/max_terminated_length": 2800.0,
      "completions/mean_length": 2166.28125,
      "completions/mean_terminated_length": 2166.28125,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.11421543313190341,
      "epoch": 0.00224,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.0997257232666016,
      "kl": 0.001993668673094362,
      "learning_rate": 6.171428571428571e-06,
      "loss": 0.0093,
      "num_tokens": 2152509.0,
      "reward": 0.4090625047683716,
      "reward_std": 0.14099711179733276,
      "rewards/rollout_reward_func/mean": 0.4090625047683716,
      "rewards/rollout_reward_func/std": 0.3153631389141083,
      "sampling/importance_sampling_ratio/max": 1.4471888542175293,
      "sampling/importance_sampling_ratio/mean": 0.9940881133079529,
      "sampling/importance_sampling_ratio/min": 0.36628207564353943,
      "sampling/sampling_logp_difference/max": 0.4437229633331299,
      "sampling/sampling_logp_difference/mean": 0.01664617471396923,
      "step": 28,
      "step_time": 16.654660168999953
    },
    {
      "clip_ratio/high_max": 0.014613970648497343,
      "clip_ratio/high_mean": 0.009043096331879497,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.010996221215464175,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2427.0,
      "completions/max_terminated_length": 2427.0,
      "completions/mean_length": 1541.1875,
      "completions/mean_terminated_length": 1541.1875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.11991730704903603,
      "epoch": 0.00232,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.753915548324585,
      "kl": 0.004232257339026546,
      "learning_rate": 6.4e-06,
      "loss": -0.0561,
      "num_tokens": 2213064.0,
      "reward": 0.7221875190734863,
      "reward_std": 0.28713130950927734,
      "rewards/rollout_reward_func/mean": 0.7221875190734863,
      "rewards/rollout_reward_func/std": 0.47223374247550964,
      "sampling/importance_sampling_ratio/max": 1.8491572141647339,
      "sampling/importance_sampling_ratio/mean": 0.9664063453674316,
      "sampling/importance_sampling_ratio/min": 0.2533723711967468,
      "sampling/sampling_logp_difference/max": 0.7475757598876953,
      "sampling/sampling_logp_difference/mean": 0.02090834453701973,
      "step": 29,
      "step_time": 14.724566217999836
    },
    {
      "clip_ratio/high_max": 0.011101973708719015,
      "clip_ratio/high_mean": 0.0055509868543595076,
      "clip_ratio/low_mean": 0.0036764706019312143,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009227457456290722,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2795.0,
      "completions/max_terminated_length": 2795.0,
      "completions/mean_length": 1690.28125,
      "completions/mean_terminated_length": 1690.28125,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.15091887768357992,
      "epoch": 0.0024,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 1.5955613851547241,
      "kl": 0.003297277187812142,
      "learning_rate": 6.628571428571428e-06,
      "loss": -0.0312,
      "num_tokens": 2278894.0,
      "reward": 0.5768749713897705,
      "reward_std": 0.2879711389541626,
      "rewards/rollout_reward_func/mean": 0.5768749713897705,
      "rewards/rollout_reward_func/std": 0.4466031789779663,
      "sampling/importance_sampling_ratio/max": 2.0268845558166504,
      "sampling/importance_sampling_ratio/mean": 0.9738575220108032,
      "sampling/importance_sampling_ratio/min": 0.3578207492828369,
      "sampling/sampling_logp_difference/max": 0.7220923900604248,
      "sampling/sampling_logp_difference/mean": 0.023466479033231735,
      "step": 30,
      "step_time": 15.794334728999956
    },
    {
      "clip_ratio/high_max": 0.008928571827709675,
      "clip_ratio/high_mean": 0.006302521098405123,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006302521098405123,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2791.0,
      "completions/max_terminated_length": 2791.0,
      "completions/mean_length": 1802.28125,
      "completions/mean_terminated_length": 1802.28125,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.1271415469236672,
      "epoch": 0.00248,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 0.5290101170539856,
      "kl": 0.007829649756786239,
      "learning_rate": 6.857142857142856e-06,
      "loss": 0.0219,
      "num_tokens": 2348595.0,
      "reward": 0.6468750238418579,
      "reward_std": 0.15625,
      "rewards/rollout_reward_func/mean": 0.6468750238418579,
      "rewards/rollout_reward_func/std": 0.4312205910682678,
      "sampling/importance_sampling_ratio/max": 1.4284578561782837,
      "sampling/importance_sampling_ratio/mean": 1.008836269378662,
      "sampling/importance_sampling_ratio/min": 0.5545295476913452,
      "sampling/sampling_logp_difference/max": 0.9234024286270142,
      "sampling/sampling_logp_difference/mean": 0.020626772195100784,
      "step": 31,
      "step_time": 16.82306190600002
    },
    {
      "clip_ratio/high_max": 0.010130719048902392,
      "clip_ratio/high_mean": 0.005065359524451196,
      "clip_ratio/low_mean": 0.0036210318794474006,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008686391403898597,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2450.0,
      "completions/max_terminated_length": 2450.0,
      "completions/mean_length": 1986.28125,
      "completions/mean_terminated_length": 1986.28125,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.13206067122519016,
      "epoch": 0.00256,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.3528786897659302,
      "kl": 0.008182306781236548,
      "learning_rate": 7.085714285714285e-06,
      "loss": -0.0853,
      "num_tokens": 2424934.0,
      "reward": 0.4606249928474426,
      "reward_std": 0.15091878175735474,
      "rewards/rollout_reward_func/mean": 0.4606249928474426,
      "rewards/rollout_reward_func/std": 0.3846149146556854,
      "sampling/importance_sampling_ratio/max": 2.71755313873291,
      "sampling/importance_sampling_ratio/mean": 1.051027774810791,
      "sampling/importance_sampling_ratio/min": 0.38737601041793823,
      "sampling/sampling_logp_difference/max": 0.7733535766601562,
      "sampling/sampling_logp_difference/mean": 0.020959284156560898,
      "step": 32,
      "step_time": 15.57071794400008
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0022321429569274187,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2803.0,
      "completions/max_terminated_length": 2803.0,
      "completions/mean_length": 2106.9375,
      "completions/mean_terminated_length": 2106.9375,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.12445190898142755,
      "epoch": 0.00264,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.12888294458389282,
      "kl": 0.003734915575478226,
      "learning_rate": 7.314285714285714e-06,
      "loss": -0.0017,
      "num_tokens": 2505564.0,
      "reward": 0.4637500047683716,
      "reward_std": 0.13466876745224,
      "rewards/rollout_reward_func/mean": 0.4637500047683716,
      "rewards/rollout_reward_func/std": 0.37349048256874084,
      "sampling/importance_sampling_ratio/max": 2.2378056049346924,
      "sampling/importance_sampling_ratio/mean": 1.0552072525024414,
      "sampling/importance_sampling_ratio/min": 0.3374383747577667,
      "sampling/sampling_logp_difference/max": 1.084688663482666,
      "sampling/sampling_logp_difference/mean": 0.015957504510879517,
      "step": 33,
      "step_time": 16.374967034000065
    },
    {
      "clip_ratio/high_max": 0.004464285913854837,
      "clip_ratio/high_mean": 0.0022321429569274187,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004464285913854837,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2416.0,
      "completions/max_terminated_length": 2416.0,
      "completions/mean_length": 1479.28125,
      "completions/mean_terminated_length": 1479.28125,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.07319290563464165,
      "epoch": 0.00272,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.5905581712722778,
      "kl": 0.02232863545214059,
      "learning_rate": 7.542857142857142e-06,
      "loss": 0.0285,
      "num_tokens": 2564043.0,
      "reward": 0.7212499976158142,
      "reward_std": 0.31069982051849365,
      "rewards/rollout_reward_func/mean": 0.7212499976158142,
      "rewards/rollout_reward_func/std": 0.45388466119766235,
      "sampling/importance_sampling_ratio/max": 1.5257185697555542,
      "sampling/importance_sampling_ratio/mean": 0.9286473989486694,
      "sampling/importance_sampling_ratio/min": 0.35652607679367065,
      "sampling/sampling_logp_difference/max": 1.0259580612182617,
      "sampling/sampling_logp_difference/mean": 0.0132124163210392,
      "step": 34,
      "step_time": 14.42571913799975
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2806.0,
      "completions/max_terminated_length": 2806.0,
      "completions/mean_length": 1752.5,
      "completions/mean_terminated_length": 1752.5,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.09324552165344357,
      "epoch": 0.0028,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.717671811580658,
      "kl": 0.004631695612260955,
      "learning_rate": 7.771428571428572e-06,
      "loss": 0.0108,
      "num_tokens": 2632179.0,
      "reward": 0.4125000238418579,
      "reward_std": 0.26933753490448,
      "rewards/rollout_reward_func/mean": 0.4125000238418579,
      "rewards/rollout_reward_func/std": 0.36455005407333374,
      "sampling/importance_sampling_ratio/max": 1.790269136428833,
      "sampling/importance_sampling_ratio/mean": 1.1154439449310303,
      "sampling/importance_sampling_ratio/min": 0.7148804068565369,
      "sampling/sampling_logp_difference/max": 0.596367359161377,
      "sampling/sampling_logp_difference/mean": 0.01612972654402256,
      "step": 35,
      "step_time": 16.94071342400025
    },
    {
      "clip_ratio/high_max": 0.003289473708719015,
      "clip_ratio/high_mean": 0.0016447368543595076,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0016447368543595076,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2780.0,
      "completions/max_terminated_length": 2780.0,
      "completions/mean_length": 2210.65625,
      "completions/mean_terminated_length": 2210.65625,
      "completions/min_length": 1565.0,
      "completions/min_terminated_length": 1565.0,
      "entropy": 0.13052229024469852,
      "epoch": 0.00288,
      "frac_reward_zero_std": 0.875,
      "grad_norm": 1.915448784828186,
      "kl": 0.014049735953449272,
      "learning_rate": 8e-06,
      "loss": 0.0114,
      "num_tokens": 2716101.0,
      "reward": 0.3787500262260437,
      "reward_std": 0.0624999962747097,
      "rewards/rollout_reward_func/mean": 0.3787500262260437,
      "rewards/rollout_reward_func/std": 0.26268768310546875,
      "sampling/importance_sampling_ratio/max": 1.6097471714019775,
      "sampling/importance_sampling_ratio/mean": 0.8499077558517456,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 24.667917251586914,
      "sampling/sampling_logp_difference/mean": 0.08357222378253937,
      "step": 36,
      "step_time": 16.647620255999527
    },
    {
      "clip_ratio/high_max": 0.013392857741564512,
      "clip_ratio/high_mean": 0.006696428870782256,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006696428870782256,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2787.0,
      "completions/max_terminated_length": 2787.0,
      "completions/mean_length": 1735.125,
      "completions/mean_terminated_length": 1735.125,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.08023441676050425,
      "epoch": 0.00296,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.0067864656448364,
      "kl": 0.014214006034308113,
      "learning_rate": 7.999999976246485e-06,
      "loss": 0.005,
      "num_tokens": 2783558.0,
      "reward": 0.5481250286102295,
      "reward_std": 0.2473391890525818,
      "rewards/rollout_reward_func/mean": 0.5481250286102295,
      "rewards/rollout_reward_func/std": 0.41463109850883484,
      "sampling/importance_sampling_ratio/max": 1.9839459657669067,
      "sampling/importance_sampling_ratio/mean": 1.0556774139404297,
      "sampling/importance_sampling_ratio/min": 0.5570288896560669,
      "sampling/sampling_logp_difference/max": 0.7343063354492188,
      "sampling/sampling_logp_difference/mean": 0.01291065476834774,
      "step": 37,
      "step_time": 15.852365188000249
    },
    {
      "clip_ratio/high_max": 0.013429548125714064,
      "clip_ratio/high_mean": 0.008277274086140096,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008277274086140096,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2790.0,
      "completions/max_terminated_length": 2790.0,
      "completions/mean_length": 1825.5625,
      "completions/mean_terminated_length": 1825.5625,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.07684489572420716,
      "epoch": 0.00304,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 0.9191060662269592,
      "kl": 0.008681207757035736,
      "learning_rate": 7.999999904985944e-06,
      "loss": -0.0211,
      "num_tokens": 2854054.0,
      "reward": 0.6575000286102295,
      "reward_std": 0.32216876745224,
      "rewards/rollout_reward_func/mean": 0.6575000286102295,
      "rewards/rollout_reward_func/std": 0.46569401025772095,
      "sampling/importance_sampling_ratio/max": 1.8603460788726807,
      "sampling/importance_sampling_ratio/mean": 1.0407439470291138,
      "sampling/importance_sampling_ratio/min": 0.5873942971229553,
      "sampling/sampling_logp_difference/max": 0.6204257011413574,
      "sampling/sampling_logp_difference/mean": 0.011706141754984856,
      "step": 38,
      "step_time": 16.086839031999943
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2800.0,
      "completions/max_terminated_length": 2800.0,
      "completions/mean_length": 2006.0625,
      "completions/mean_terminated_length": 2006.0625,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.09726127330213785,
      "epoch": 0.00312,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.442636489868164,
      "kl": 0.01900345625472255,
      "learning_rate": 7.999999786218377e-06,
      "loss": -0.0229,
      "num_tokens": 2930928.0,
      "reward": 0.42125001549720764,
      "reward_std": 0.13466876745224,
      "rewards/rollout_reward_func/mean": 0.42125001549720764,
      "rewards/rollout_reward_func/std": 0.3237656354904175,
      "sampling/importance_sampling_ratio/max": 1.6864854097366333,
      "sampling/importance_sampling_ratio/mean": 0.9331543445587158,
      "sampling/importance_sampling_ratio/min": 0.17830577492713928,
      "sampling/sampling_logp_difference/max": 0.9498655796051025,
      "sampling/sampling_logp_difference/mean": 0.01994011551141739,
      "step": 39,
      "step_time": 17.24544241999979
    },
    {
      "clip_ratio/high_max": 0.013523391913622618,
      "clip_ratio/high_mean": 0.006761695956811309,
      "clip_ratio/low_mean": 0.0016447368543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008406432811170816,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2440.0,
      "completions/max_terminated_length": 2440.0,
      "completions/mean_length": 1899.96875,
      "completions/mean_terminated_length": 1899.96875,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.09562211390584707,
      "epoch": 0.0032,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.7564519047737122,
      "kl": 0.012280215043574572,
      "learning_rate": 7.999999619943787e-06,
      "loss": 0.0388,
      "num_tokens": 3004071.0,
      "reward": 0.53125,
      "reward_std": 0.13466876745224,
      "rewards/rollout_reward_func/mean": 0.53125,
      "rewards/rollout_reward_func/std": 0.40266650915145874,
      "sampling/importance_sampling_ratio/max": 1.7533916234970093,
      "sampling/importance_sampling_ratio/mean": 1.012838363647461,
      "sampling/importance_sampling_ratio/min": 0.40302401781082153,
      "sampling/sampling_logp_difference/max": 0.6195348501205444,
      "sampling/sampling_logp_difference/mean": 0.016937807202339172,
      "step": 40,
      "step_time": 15.278612154999792
    },
    {
      "clip_ratio/high_max": 0.013322473270818591,
      "clip_ratio/high_mean": 0.0066612366354092956,
      "clip_ratio/low_mean": 0.0016447368543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008305973489768803,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2818.0,
      "completions/max_terminated_length": 2818.0,
      "completions/mean_length": 2140.78125,
      "completions/mean_terminated_length": 2140.78125,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.10391843365505338,
      "epoch": 0.00328,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 0.684903621673584,
      "kl": 0.0072205948672490194,
      "learning_rate": 7.999999406162173e-06,
      "loss": 0.0078,
      "num_tokens": 3085711.0,
      "reward": 0.41718751192092896,
      "reward_std": 0.14279377460479736,
      "rewards/rollout_reward_func/mean": 0.41718751192092896,
      "rewards/rollout_reward_func/std": 0.33007559180259705,
      "sampling/importance_sampling_ratio/max": 1.5918241739273071,
      "sampling/importance_sampling_ratio/mean": 0.900505006313324,
      "sampling/importance_sampling_ratio/min": 0.29988786578178406,
      "sampling/sampling_logp_difference/max": 1.036886215209961,
      "sampling/sampling_logp_difference/mean": 0.019227981567382812,
      "step": 41,
      "step_time": 16.941576514999497
    },
    {
      "clip_ratio/high_max": 0.0032051282469183207,
      "clip_ratio/high_mean": 0.0016025641234591603,
      "clip_ratio/low_mean": 0.0035156250232830644,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005118189146742225,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2780.0,
      "completions/max_terminated_length": 2780.0,
      "completions/mean_length": 1944.25,
      "completions/mean_terminated_length": 1944.25,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.09584560617804527,
      "epoch": 0.00336,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 2.3526816368103027,
      "kl": 0.027870278747286648,
      "learning_rate": 7.999999144873542e-06,
      "loss": 0.1103,
      "num_tokens": 3160194.0,
      "reward": 0.4490624964237213,
      "reward_std": 0.20263297855854034,
      "rewards/rollout_reward_func/mean": 0.4490624964237213,
      "rewards/rollout_reward_func/std": 0.36338335275650024,
      "sampling/importance_sampling_ratio/max": 2.6562483310699463,
      "sampling/importance_sampling_ratio/mean": 1.036879062652588,
      "sampling/importance_sampling_ratio/min": 0.24595271050930023,
      "sampling/sampling_logp_difference/max": 0.9235069751739502,
      "sampling/sampling_logp_difference/mean": 0.02009188011288643,
      "step": 42,
      "step_time": 16.650967574000333
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0066964286379516125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0066964286379516125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2791.0,
      "completions/max_terminated_length": 2791.0,
      "completions/mean_length": 1675.03125,
      "completions/mean_terminated_length": 1675.03125,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.12920551793649793,
      "epoch": 0.00344,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.3091895580291748,
      "kl": 0.03491167013999075,
      "learning_rate": 7.999998836077897e-06,
      "loss": 0.0888,
      "num_tokens": 3225903.0,
      "reward": 0.5174999833106995,
      "reward_std": 0.3608438968658447,
      "rewards/rollout_reward_func/mean": 0.5174999833106995,
      "rewards/rollout_reward_func/std": 0.43675488233566284,
      "sampling/importance_sampling_ratio/max": 2.373917579650879,
      "sampling/importance_sampling_ratio/mean": 1.0001801252365112,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.3759169578552246,
      "sampling/sampling_logp_difference/mean": 0.02347693033516407,
      "step": 43,
      "step_time": 17.192637601000115
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2798.0,
      "completions/max_terminated_length": 2798.0,
      "completions/mean_length": 2095.3125,
      "completions/mean_terminated_length": 2095.3125,
      "completions/min_length": 1562.0,
      "completions/min_terminated_length": 1562.0,
      "entropy": 0.09558335272595286,
      "epoch": 0.00352,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.6341590881347656,
      "kl": 0.02431841316865757,
      "learning_rate": 7.99999847977524e-06,
      "loss": -0.0206,
      "num_tokens": 3305945.0,
      "reward": 0.5049999952316284,
      "reward_std": 0.14433754980564117,
      "rewards/rollout_reward_func/mean": 0.5049999952316284,
      "rewards/rollout_reward_func/std": 0.3978976607322693,
      "sampling/importance_sampling_ratio/max": 2.0796568393707275,
      "sampling/importance_sampling_ratio/mean": 0.8876084089279175,
      "sampling/importance_sampling_ratio/min": 0.34211352467536926,
      "sampling/sampling_logp_difference/max": 1.0857441425323486,
      "sampling/sampling_logp_difference/mean": 0.01929028518497944,
      "step": 44,
      "step_time": 16.607955621999963
    },
    {
      "clip_ratio/high_max": 0.009783434681594372,
      "clip_ratio/high_mean": 0.004891717340797186,
      "clip_ratio/low_mean": 0.0034007353242486715,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008292452665045857,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2816.0,
      "completions/max_terminated_length": 2816.0,
      "completions/mean_length": 2426.71875,
      "completions/mean_terminated_length": 2426.71875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.13120493851602077,
      "epoch": 0.0036,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.2010557651519775,
      "kl": 0.014120981490123086,
      "learning_rate": 7.999998075965583e-06,
      "loss": -0.0277,
      "num_tokens": 3397449.0,
      "reward": 0.3384374976158142,
      "reward_std": 0.08029378205537796,
      "rewards/rollout_reward_func/mean": 0.3384374976158142,
      "rewards/rollout_reward_func/std": 0.21492847800254822,
      "sampling/importance_sampling_ratio/max": 2.0331103801727295,
      "sampling/importance_sampling_ratio/mean": 1.036205768585205,
      "sampling/importance_sampling_ratio/min": 0.44029006361961365,
      "sampling/sampling_logp_difference/max": 0.8087775707244873,
      "sampling/sampling_logp_difference/mean": 0.021842796355485916,
      "step": 45,
      "step_time": 17.039232532999677
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.004464285913854837,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004464285913854837,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2820.0,
      "completions/max_terminated_length": 2820.0,
      "completions/mean_length": 2035.78125,
      "completions/mean_terminated_length": 2035.78125,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.10772825870662928,
      "epoch": 0.00368,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.6874250173568726,
      "kl": 0.02060257241828367,
      "learning_rate": 7.99999762464893e-06,
      "loss": 0.0037,
      "num_tokens": 3475466.0,
      "reward": 0.35374999046325684,
      "reward_std": 0.13466878235340118,
      "rewards/rollout_reward_func/mean": 0.35374999046325684,
      "rewards/rollout_reward_func/std": 0.26349693536758423,
      "sampling/importance_sampling_ratio/max": 2.4506301879882812,
      "sampling/importance_sampling_ratio/mean": 1.0585436820983887,
      "sampling/importance_sampling_ratio/min": 0.20336686074733734,
      "sampling/sampling_logp_difference/max": 1.0288989543914795,
      "sampling/sampling_logp_difference/mean": 0.019599031656980515,
      "step": 46,
      "step_time": 17.008759735000012
    },
    {
      "clip_ratio/high_max": 0.007352941203862429,
      "clip_ratio/high_mean": 0.0036764706019312143,
      "clip_ratio/low_mean": 0.0018382353009656072,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0055147059028968215,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2796.0,
      "completions/max_terminated_length": 2796.0,
      "completions/mean_length": 1804.875,
      "completions/mean_terminated_length": 1804.875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.11447951383888721,
      "epoch": 0.00376,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.9577285051345825,
      "kl": 0.0146886624279432,
      "learning_rate": 7.999997125825284e-06,
      "loss": 0.018,
      "num_tokens": 3545642.0,
      "reward": 0.550000011920929,
      "reward_std": 0.25521132349967957,
      "rewards/rollout_reward_func/mean": 0.550000011920929,
      "rewards/rollout_reward_func/std": 0.4146043360233307,
      "sampling/importance_sampling_ratio/max": 2.3037173748016357,
      "sampling/importance_sampling_ratio/mean": 1.043008804321289,
      "sampling/importance_sampling_ratio/min": 0.5624377727508545,
      "sampling/sampling_logp_difference/max": 0.6249582767486572,
      "sampling/sampling_logp_difference/mean": 0.017069321125745773,
      "step": 47,
      "step_time": 17.18442160299992
    },
    {
      "clip_ratio/high_max": 0.011488970601931214,
      "clip_ratio/high_mean": 0.005744485300965607,
      "clip_ratio/low_mean": 0.003968254080973566,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.009712739381939173,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2804.0,
      "completions/max_terminated_length": 2804.0,
      "completions/mean_length": 1970.9375,
      "completions/mean_terminated_length": 1970.9375,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.09903696551918983,
      "epoch": 0.00384,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.1492125988006592,
      "kl": 0.018403344380203635,
      "learning_rate": 7.999996579494655e-06,
      "loss": 0.0456,
      "num_tokens": 3621220.0,
      "reward": 0.4325000047683716,
      "reward_std": 0.19716876745224,
      "rewards/rollout_reward_func/mean": 0.4325000047683716,
      "rewards/rollout_reward_func/std": 0.3565334677696228,
      "sampling/importance_sampling_ratio/max": 2.369974136352539,
      "sampling/importance_sampling_ratio/mean": 1.0478521585464478,
      "sampling/importance_sampling_ratio/min": 0.11733747273683548,
      "sampling/sampling_logp_difference/max": 1.4917361736297607,
      "sampling/sampling_logp_difference/mean": 0.022218499332666397,
      "step": 48,
      "step_time": 16.41912825899999
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2804.0,
      "completions/max_terminated_length": 2804.0,
      "completions/mean_length": 2252.875,
      "completions/mean_terminated_length": 2252.875,
      "completions/min_length": 1565.0,
      "completions/min_terminated_length": 1565.0,
      "entropy": 0.09296703850850463,
      "epoch": 0.00392,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.5641649961471558,
      "kl": 0.03362530213780701,
      "learning_rate": 7.999995985657054e-06,
      "loss": -0.0171,
      "num_tokens": 3706955.0,
      "reward": 0.42624998092651367,
      "reward_std": 0.13466876745224,
      "rewards/rollout_reward_func/mean": 0.42624998092651367,
      "rewards/rollout_reward_func/std": 0.3314265012741089,
      "sampling/importance_sampling_ratio/max": 1.5233134031295776,
      "sampling/importance_sampling_ratio/mean": 0.9442777633666992,
      "sampling/importance_sampling_ratio/min": 0.2804383933544159,
      "sampling/sampling_logp_difference/max": 1.1384481191635132,
      "sampling/sampling_logp_difference/mean": 0.01721033826470375,
      "step": 49,
      "step_time": 16.923561193999603
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2820.0,
      "completions/max_terminated_length": 2820.0,
      "completions/mean_length": 2463.8125,
      "completions/mean_terminated_length": 2463.8125,
      "completions/min_length": 2040.0,
      "completions/min_terminated_length": 2040.0,
      "entropy": 0.09066143818199635,
      "epoch": 0.004,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.10026834160089493,
      "kl": 0.030927304484066553,
      "learning_rate": 7.99999534431249e-06,
      "loss": 0.0004,
      "num_tokens": 3799818.0,
      "reward": 0.30000001192092896,
      "reward_std": 0.0,
      "rewards/rollout_reward_func/mean": 0.30000001192092896,
      "rewards/rollout_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.050495147705078,
      "sampling/importance_sampling_ratio/mean": 1.0762633085250854,
      "sampling/importance_sampling_ratio/min": 0.29474106431007385,
      "sampling/sampling_logp_difference/max": 1.1381915807724,
      "sampling/sampling_logp_difference/mean": 0.01666702888906002,
      "step": 50,
      "step_time": 17.35131004799996
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0018382353009656072,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0018382353009656072,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2786.0,
      "completions/max_terminated_length": 2786.0,
      "completions/mean_length": 1773.84375,
      "completions/mean_terminated_length": 1773.84375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.06475613545626402,
      "epoch": 0.00408,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 3.30958890914917,
      "kl": 0.015178573405137286,
      "learning_rate": 7.99999465546097e-06,
      "loss": 0.0343,
      "num_tokens": 3868368.0,
      "reward": 0.49562498927116394,
      "reward_std": 0.1848391890525818,
      "rewards/rollout_reward_func/mean": 0.49562498927116394,
      "rewards/rollout_reward_func/std": 0.4042271077632904,
      "sampling/importance_sampling_ratio/max": 2.036992311477661,
      "sampling/importance_sampling_ratio/mean": 1.0444798469543457,
      "sampling/importance_sampling_ratio/min": 0.6844988465309143,
      "sampling/sampling_logp_difference/max": 0.4961543083190918,
      "sampling/sampling_logp_difference/mean": 0.009073879569768906,
      "step": 51,
      "step_time": 17.00100525200014
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2809.0,
      "completions/max_terminated_length": 2809.0,
      "completions/mean_length": 1786.375,
      "completions/mean_terminated_length": 1786.375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.08096808800473809,
      "epoch": 0.00416,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.6158044338226318,
      "kl": 0.020677090127719566,
      "learning_rate": 7.99999391910251e-06,
      "loss": -0.0195,
      "num_tokens": 3937756.0,
      "reward": 0.4912499785423279,
      "reward_std": 0.33183753490448,
      "rewards/rollout_reward_func/mean": 0.4912499785423279,
      "rewards/rollout_reward_func/std": 0.42849886417388916,
      "sampling/importance_sampling_ratio/max": 1.5450013875961304,
      "sampling/importance_sampling_ratio/mean": 0.9464795589447021,
      "sampling/importance_sampling_ratio/min": 0.39816370606422424,
      "sampling/sampling_logp_difference/max": 1.0278459787368774,
      "sampling/sampling_logp_difference/mean": 0.015709228813648224,
      "step": 52,
      "step_time": 16.014014809000173
    },
    {
      "clip_ratio/high_max": 0.003289473708719015,
      "clip_ratio/high_mean": 0.0016447368543595076,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0016447368543595076,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2799.0,
      "completions/max_terminated_length": 2799.0,
      "completions/mean_length": 1632.4375,
      "completions/mean_terminated_length": 1632.4375,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.06091495987493545,
      "epoch": 0.00424,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.7226957678794861,
      "kl": 0.017386693391017616,
      "learning_rate": 7.999993135237117e-06,
      "loss": 0.0201,
      "num_tokens": 4001420.0,
      "reward": 0.6793749928474426,
      "reward_std": 0.2570079565048218,
      "rewards/rollout_reward_func/mean": 0.6793749928474426,
      "rewards/rollout_reward_func/std": 0.46435481309890747,
      "sampling/importance_sampling_ratio/max": 1.579625129699707,
      "sampling/importance_sampling_ratio/mean": 0.9135901927947998,
      "sampling/importance_sampling_ratio/min": 0.35018137097358704,
      "sampling/sampling_logp_difference/max": 1.1071686744689941,
      "sampling/sampling_logp_difference/mean": 0.015183830633759499,
      "step": 53,
      "step_time": 15.960780017000161
    },
    {
      "clip_ratio/high_max": 0.00390625,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00390625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2809.0,
      "completions/max_terminated_length": 2809.0,
      "completions/mean_length": 1589.6875,
      "completions/mean_terminated_length": 1589.6875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.054683255730196834,
      "epoch": 0.00432,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.0967762470245361,
      "kl": 0.021693169721402228,
      "learning_rate": 7.999992303864804e-06,
      "loss": -0.0452,
      "num_tokens": 4063548.0,
      "reward": 0.7262499928474426,
      "reward_std": 0.3071783781051636,
      "rewards/rollout_reward_func/mean": 0.7262499928474426,
      "rewards/rollout_reward_func/std": 0.45173320174217224,
      "sampling/importance_sampling_ratio/max": 1.3698084354400635,
      "sampling/importance_sampling_ratio/mean": 0.9228720664978027,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.3379874229431152,
      "sampling/sampling_logp_difference/mean": 0.013593094423413277,
      "step": 54,
      "step_time": 16.424249653999823
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0057151151122525334,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0057151151122525334,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2435.0,
      "completions/max_terminated_length": 2435.0,
      "completions/mean_length": 1791.34375,
      "completions/mean_terminated_length": 1791.34375,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.06585042458027601,
      "epoch": 0.0044,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.214758038520813,
      "kl": 0.024209468625485897,
      "learning_rate": 7.999991424985586e-06,
      "loss": 0.0112,
      "num_tokens": 4132756.0,
      "reward": 0.4637500047683716,
      "reward_std": 0.19184717535972595,
      "rewards/rollout_reward_func/mean": 0.4637500047683716,
      "rewards/rollout_reward_func/std": 0.35306718945503235,
      "sampling/importance_sampling_ratio/max": 2.3594348430633545,
      "sampling/importance_sampling_ratio/mean": 1.098282814025879,
      "sampling/importance_sampling_ratio/min": 0.3494420647621155,
      "sampling/sampling_logp_difference/max": 1.4460781812667847,
      "sampling/sampling_logp_difference/mean": 0.01887095905840397,
      "step": 55,
      "step_time": 15.803478434999533
    },
    {
      "clip_ratio/high_max": 0.003289473708719015,
      "clip_ratio/high_mean": 0.0016447368543595076,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0016447368543595076,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2794.0,
      "completions/max_terminated_length": 2794.0,
      "completions/mean_length": 2308.09375,
      "completions/mean_terminated_length": 2308.09375,
      "completions/min_length": 1568.0,
      "completions/min_terminated_length": 1568.0,
      "entropy": 0.09696381096728146,
      "epoch": 0.00448,
      "frac_reward_zero_std": 0.875,
      "grad_norm": 0.32451269030570984,
      "kl": 0.019486179284285754,
      "learning_rate": 7.999990498599477e-06,
      "loss": -0.0013,
      "num_tokens": 4220279.0,
      "reward": 0.3631250262260437,
      "reward_std": 0.05983918905258179,
      "rewards/rollout_reward_func/mean": 0.3631250262260437,
      "rewards/rollout_reward_func/std": 0.2257665991783142,
      "sampling/importance_sampling_ratio/max": 1.3443750143051147,
      "sampling/importance_sampling_ratio/mean": 0.9363906979560852,
      "sampling/importance_sampling_ratio/min": 0.3993731737136841,
      "sampling/sampling_logp_difference/max": 0.668013334274292,
      "sampling/sampling_logp_difference/mean": 0.01326768472790718,
      "step": 56,
      "step_time": 16.922060316999477
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2460.0,
      "completions/max_terminated_length": 2460.0,
      "completions/mean_length": 1963.34375,
      "completions/mean_terminated_length": 1963.34375,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.04914194135926664,
      "epoch": 0.00456,
      "frac_reward_zero_std": 0.875,
      "grad_norm": 0.5049991607666016,
      "kl": 0.02837482520226331,
      "learning_rate": 7.99998952470649e-06,
      "loss": -0.008,
      "num_tokens": 4295471.0,
      "reward": 0.48374998569488525,
      "reward_std": 0.0624999962747097,
      "rewards/rollout_reward_func/mean": 0.48374998569488525,
      "rewards/rollout_reward_func/std": 0.3627649247646332,
      "sampling/importance_sampling_ratio/max": 1.912903904914856,
      "sampling/importance_sampling_ratio/mean": 0.9784045219421387,
      "sampling/importance_sampling_ratio/min": 0.4301539659500122,
      "sampling/sampling_logp_difference/max": 0.8482755422592163,
      "sampling/sampling_logp_difference/mean": 0.010939370840787888,
      "step": 57,
      "step_time": 15.453092248000303
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0034007353242486715,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0034007353242486715,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2787.0,
      "completions/max_terminated_length": 2787.0,
      "completions/mean_length": 1872.28125,
      "completions/mean_terminated_length": 1872.28125,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.06335801596287638,
      "epoch": 0.00464,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 0.6874297261238098,
      "kl": 0.03325861978373723,
      "learning_rate": 7.999988503306642e-06,
      "loss": -0.0215,
      "num_tokens": 4367703.0,
      "reward": 0.4325000047683716,
      "reward_std": 0.19716876745224,
      "rewards/rollout_reward_func/mean": 0.4325000047683716,
      "rewards/rollout_reward_func/std": 0.3528958261013031,
      "sampling/importance_sampling_ratio/max": 1.3908125162124634,
      "sampling/importance_sampling_ratio/mean": 0.8833715319633484,
      "sampling/importance_sampling_ratio/min": 1.9220989599944005e-07,
      "sampling/sampling_logp_difference/max": 13.085790634155273,
      "sampling/sampling_logp_difference/mean": 0.04347304627299309,
      "step": 58,
      "step_time": 16.225704187000247
    },
    {
      "clip_ratio/high_max": 0.004464285913854837,
      "clip_ratio/high_mean": 0.0022321429569274187,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0022321429569274187,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2810.0,
      "completions/max_terminated_length": 2810.0,
      "completions/mean_length": 2212.5,
      "completions/mean_terminated_length": 2212.5,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.0681739835999906,
      "epoch": 0.00472,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.8530667424201965,
      "kl": 0.03184444556973176,
      "learning_rate": 7.999987434399948e-06,
      "loss": 0.0114,
      "num_tokens": 4452096.0,
      "reward": 0.4006250202655792,
      "reward_std": 0.13200798630714417,
      "rewards/rollout_reward_func/mean": 0.4006250202655792,
      "rewards/rollout_reward_func/std": 0.29461774230003357,
      "sampling/importance_sampling_ratio/max": 1.6479169130325317,
      "sampling/importance_sampling_ratio/mean": 1.004500150680542,
      "sampling/importance_sampling_ratio/min": 0.30276352167129517,
      "sampling/sampling_logp_difference/max": 1.1370731592178345,
      "sampling/sampling_logp_difference/mean": 0.014959340915083885,
      "step": 59,
      "step_time": 17.880038248999654
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0031250000465661287,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0031250000465661287,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2438.0,
      "completions/max_terminated_length": 2438.0,
      "completions/mean_length": 2130.46875,
      "completions/mean_terminated_length": 2130.46875,
      "completions/min_length": 1567.0,
      "completions/min_terminated_length": 1567.0,
      "entropy": 0.055047230795025826,
      "epoch": 0.0048,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.3532449007034302,
      "kl": 0.01998938515316695,
      "learning_rate": 7.999986317986426e-06,
      "loss": -0.0728,
      "num_tokens": 4533067.0,
      "reward": 0.3746874928474426,
      "reward_std": 0.07062499970197678,
      "rewards/rollout_reward_func/mean": 0.3746874928474426,
      "rewards/rollout_reward_func/std": 0.26494044065475464,
      "sampling/importance_sampling_ratio/max": 1.6606436967849731,
      "sampling/importance_sampling_ratio/mean": 0.94105064868927,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 0.8746337890625,
      "sampling/sampling_logp_difference/mean": 0.013816887512803078,
      "step": 60,
      "step_time": 15.591869645000315
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0022321429569274187,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0022321429569274187,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2772.0,
      "completions/max_terminated_length": 2772.0,
      "completions/mean_length": 1693.875,
      "completions/mean_terminated_length": 1693.875,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.05530149070546031,
      "epoch": 0.00488,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.1544066667556763,
      "kl": 0.016453518153866753,
      "learning_rate": 7.999985154066091e-06,
      "loss": 0.0093,
      "num_tokens": 4598963.0,
      "reward": 0.5693750381469727,
      "reward_std": 0.32917672395706177,
      "rewards/rollout_reward_func/mean": 0.5693750381469727,
      "rewards/rollout_reward_func/std": 0.4425143301486969,
      "sampling/importance_sampling_ratio/max": 2.6382031440734863,
      "sampling/importance_sampling_ratio/mean": 1.0790200233459473,
      "sampling/importance_sampling_ratio/min": 0.7093254327774048,
      "sampling/sampling_logp_difference/max": 0.6202226877212524,
      "sampling/sampling_logp_difference/mean": 0.011032961308956146,
      "step": 61,
      "step_time": 16.082178944999896
    },
    {
      "clip_ratio/high_max": 0.0062500000931322575,
      "clip_ratio/high_mean": 0.0031250000465661287,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0031250000465661287,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2806.0,
      "completions/max_terminated_length": 2806.0,
      "completions/mean_length": 2304.9375,
      "completions/mean_terminated_length": 2304.9375,
      "completions/min_length": 1559.0,
      "completions/min_terminated_length": 1559.0,
      "entropy": 0.06317996443249285,
      "epoch": 0.00496,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.9210407137870789,
      "kl": 0.019177823327481747,
      "learning_rate": 7.999983942638965e-06,
      "loss": -0.0262,
      "num_tokens": 4686091.0,
      "reward": 0.3434374928474426,
      "reward_std": 0.08029377460479736,
      "rewards/rollout_reward_func/mean": 0.3434374928474426,
      "rewards/rollout_reward_func/std": 0.22245851159095764,
      "sampling/importance_sampling_ratio/max": 1.911454677581787,
      "sampling/importance_sampling_ratio/mean": 0.9809565544128418,
      "sampling/importance_sampling_ratio/min": 0.25244244933128357,
      "sampling/sampling_logp_difference/max": 0.9257916212081909,
      "sampling/sampling_logp_difference/mean": 0.013646715320646763,
      "step": 62,
      "step_time": 16.97750502200006
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2421.0,
      "completions/max_terminated_length": 2421.0,
      "completions/mean_length": 1974.96875,
      "completions/mean_terminated_length": 1974.96875,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.04369071568362415,
      "epoch": 0.00504,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.14146260917186737,
      "kl": 0.05203759076539427,
      "learning_rate": 7.999982683705066e-06,
      "loss": 0.0006,
      "num_tokens": 4761655.0,
      "reward": 0.2800000011920929,
      "reward_std": 0.0,
      "rewards/rollout_reward_func/mean": 0.2800000011920929,
      "rewards/rollout_reward_func/std": 0.05376172438263893,
      "sampling/importance_sampling_ratio/max": 2.2244811058044434,
      "sampling/importance_sampling_ratio/mean": 1.0279231071472168,
      "sampling/importance_sampling_ratio/min": 0.3840067684650421,
      "sampling/sampling_logp_difference/max": 0.9096496105194092,
      "sampling/sampling_logp_difference/mean": 0.010698029771447182,
      "step": 63,
      "step_time": 16.219797216000416
    },
    {
      "clip_ratio/high_max": 0.015190972248092294,
      "clip_ratio/high_mean": 0.007595486124046147,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007595486124046147,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2815.0,
      "completions/max_terminated_length": 2815.0,
      "completions/mean_length": 1982.6875,
      "completions/mean_terminated_length": 1982.6875,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.07828675024211407,
      "epoch": 0.00512,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.7813690304756165,
      "kl": 0.021903582994127646,
      "learning_rate": 7.999981377264413e-06,
      "loss": 0.0011,
      "num_tokens": 4838046.0,
      "reward": 0.5843750238418579,
      "reward_std": 0.22841876745224,
      "rewards/rollout_reward_func/mean": 0.5843750238418579,
      "rewards/rollout_reward_func/std": 0.4214792251586914,
      "sampling/importance_sampling_ratio/max": 1.827079176902771,
      "sampling/importance_sampling_ratio/mean": 1.0283104181289673,
      "sampling/importance_sampling_ratio/min": 0.25544387102127075,
      "sampling/sampling_logp_difference/max": 1.0380005836486816,
      "sampling/sampling_logp_difference/mean": 0.015725988894701004,
      "step": 64,
      "step_time": 16.373142061000408
    },
    {
      "clip_ratio/high_max": 0.0036764706019312143,
      "clip_ratio/high_mean": 0.0018382353009656072,
      "clip_ratio/low_mean": 0.0018382353009656072,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0036764706019312143,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2801.0,
      "completions/max_terminated_length": 2801.0,
      "completions/mean_length": 2009.3125,
      "completions/mean_terminated_length": 2009.3125,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.05688254698179662,
      "epoch": 0.0052,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.2757700979709625,
      "kl": 0.036635934142395854,
      "learning_rate": 7.999980023317026e-06,
      "loss": 0.0247,
      "num_tokens": 4914779.0,
      "reward": 0.45250001549720764,
      "reward_std": 0.125,
      "rewards/rollout_reward_func/mean": 0.45250001549720764,
      "rewards/rollout_reward_func/std": 0.3471450209617615,
      "sampling/importance_sampling_ratio/max": 1.4275991916656494,
      "sampling/importance_sampling_ratio/mean": 0.908840537071228,
      "sampling/importance_sampling_ratio/min": 0.23684662580490112,
      "sampling/sampling_logp_difference/max": 1.4337669610977173,
      "sampling/sampling_logp_difference/mean": 0.017221834510564804,
      "step": 65,
      "step_time": 16.79740752900011
    },
    {
      "clip_ratio/high_max": 0.0036764706019312143,
      "clip_ratio/high_mean": 0.0018382353009656072,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0018382353009656072,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2804.0,
      "completions/max_terminated_length": 2804.0,
      "completions/mean_length": 2059.75,
      "completions/mean_terminated_length": 2059.75,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.07610421534627676,
      "epoch": 0.00528,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.4526886940002441,
      "kl": 0.027375709789339453,
      "learning_rate": 7.999978621862929e-06,
      "loss": 0.0203,
      "num_tokens": 4993633.0,
      "reward": 0.4793750047683716,
      "reward_std": 0.13200797140598297,
      "rewards/rollout_reward_func/mean": 0.4793750047683716,
      "rewards/rollout_reward_func/std": 0.3699514865875244,
      "sampling/importance_sampling_ratio/max": 1.815674066543579,
      "sampling/importance_sampling_ratio/mean": 1.0289226770401,
      "sampling/importance_sampling_ratio/min": 0.5281765460968018,
      "sampling/sampling_logp_difference/max": 0.6608150005340576,
      "sampling/sampling_logp_difference/mean": 0.01253808755427599,
      "step": 66,
      "step_time": 16.794041812999467
    },
    {
      "clip_ratio/high_max": 0.011259191203862429,
      "clip_ratio/high_mean": 0.005629595601931214,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007582720601931214,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2812.0,
      "completions/max_terminated_length": 2812.0,
      "completions/mean_length": 1777.84375,
      "completions/mean_terminated_length": 1777.84375,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.06151273613795638,
      "epoch": 0.00536,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.1407759189605713,
      "kl": 0.02162244959617965,
      "learning_rate": 7.999977172902144e-06,
      "loss": -0.0017,
      "num_tokens": 5062556.0,
      "reward": 0.48000001907348633,
      "reward_std": 0.25434714555740356,
      "rewards/rollout_reward_func/mean": 0.48000001907348633,
      "rewards/rollout_reward_func/std": 0.3861806094646454,
      "sampling/importance_sampling_ratio/max": 1.6365498304367065,
      "sampling/importance_sampling_ratio/mean": 1.0415589809417725,
      "sampling/importance_sampling_ratio/min": 0.4183502197265625,
      "sampling/sampling_logp_difference/max": 0.8754826188087463,
      "sampling/sampling_logp_difference/mean": 0.014012180268764496,
      "step": 67,
      "step_time": 17.151742474000002
    },
    {
      "clip_ratio/high_max": 0.0058139534667134285,
      "clip_ratio/high_mean": 0.0029069767333567142,
      "clip_ratio/low_mean": 0.0031250000465661287,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.006031976779922843,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2778.0,
      "completions/max_terminated_length": 2778.0,
      "completions/mean_length": 2272.375,
      "completions/mean_terminated_length": 2272.375,
      "completions/min_length": 2023.0,
      "completions/min_terminated_length": 2023.0,
      "entropy": 0.08835586486384273,
      "epoch": 0.00544,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.3424510955810547,
      "kl": 0.030128376907669008,
      "learning_rate": 7.999975676434692e-06,
      "loss": -0.0562,
      "num_tokens": 5148670.0,
      "reward": 0.2918750047683716,
      "reward_std": 0.0162500012665987,
      "rewards/rollout_reward_func/mean": 0.2918750047683716,
      "rewards/rollout_reward_func/std": 0.031971510499715805,
      "sampling/importance_sampling_ratio/max": 2.0581729412078857,
      "sampling/importance_sampling_ratio/mean": 1.0349149703979492,
      "sampling/importance_sampling_ratio/min": 0.4890825152397156,
      "sampling/sampling_logp_difference/max": 0.7274646759033203,
      "sampling/sampling_logp_difference/mean": 0.019628014415502548,
      "step": 68,
      "step_time": 16.927781462999747
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.00390625,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00390625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2434.0,
      "completions/max_terminated_length": 2434.0,
      "completions/mean_length": 1876.4375,
      "completions/mean_terminated_length": 1876.4375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.0765807363204658,
      "epoch": 0.00552,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 0.9624868631362915,
      "kl": 0.05591569049283862,
      "learning_rate": 7.999974132460596e-06,
      "loss": 0.0124,
      "num_tokens": 5221035.0,
      "reward": 0.4793750047683716,
      "reward_std": 0.17558756470680237,
      "rewards/rollout_reward_func/mean": 0.4793750047683716,
      "rewards/rollout_reward_func/std": 0.3699514865875244,
      "sampling/importance_sampling_ratio/max": 1.3370881080627441,
      "sampling/importance_sampling_ratio/mean": 0.9259651899337769,
      "sampling/importance_sampling_ratio/min": 0.3385222852230072,
      "sampling/sampling_logp_difference/max": 1.2049891948699951,
      "sampling/sampling_logp_difference/mean": 0.014953669160604477,
      "step": 69,
      "step_time": 15.703423997999835
    },
    {
      "clip_ratio/high_max": 0.0069444444961845875,
      "clip_ratio/high_mean": 0.0034722222480922937,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0034722222480922937,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2437.0,
      "completions/max_terminated_length": 2437.0,
      "completions/mean_length": 1896.46875,
      "completions/mean_terminated_length": 1896.46875,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.05552901164628565,
      "epoch": 0.0056,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.2667936384677887,
      "kl": 0.014588030888262438,
      "learning_rate": 7.999972540979884e-06,
      "loss": 0.0088,
      "num_tokens": 5294017.0,
      "reward": 0.5,
      "reward_std": 0.125,
      "rewards/rollout_reward_func/mean": 0.5,
      "rewards/rollout_reward_func/std": 0.39909571409225464,
      "sampling/importance_sampling_ratio/max": 1.607146143913269,
      "sampling/importance_sampling_ratio/mean": 0.9701419472694397,
      "sampling/importance_sampling_ratio/min": 0.401731014251709,
      "sampling/sampling_logp_difference/max": 1.0000518560409546,
      "sampling/sampling_logp_difference/mean": 0.010325020179152489,
      "step": 70,
      "step_time": 15.35116849200017
    },
    {
      "clip_ratio/high_max": 0.0069659443106502295,
      "clip_ratio/high_mean": 0.0034829721553251147,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0034829721553251147,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2805.0,
      "completions/max_terminated_length": 2805.0,
      "completions/mean_length": 2170.65625,
      "completions/mean_terminated_length": 2170.65625,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.09610213804990053,
      "epoch": 0.00568,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.5360183119773865,
      "kl": 0.029458090604748577,
      "learning_rate": 7.99997090199258e-06,
      "loss": 0.0331,
      "num_tokens": 5376683.0,
      "reward": 0.515625,
      "reward_std": 0.12233918905258179,
      "rewards/rollout_reward_func/mean": 0.515625,
      "rewards/rollout_reward_func/std": 0.3893662095069885,
      "sampling/importance_sampling_ratio/max": 2.599726438522339,
      "sampling/importance_sampling_ratio/mean": 0.9910818934440613,
      "sampling/importance_sampling_ratio/min": 0.38567137718200684,
      "sampling/sampling_logp_difference/max": 0.9569785594940186,
      "sampling/sampling_logp_difference/mean": 0.02182850055396557,
      "step": 71,
      "step_time": 18.631503620000103
    },
    {
      "clip_ratio/high_max": 0.0034722222480922937,
      "clip_ratio/high_mean": 0.0017361111240461469,
      "clip_ratio/low_mean": 0.0018382353009656072,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003574346425011754,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2434.0,
      "completions/max_terminated_length": 2434.0,
      "completions/mean_length": 1713.6875,
      "completions/mean_terminated_length": 1713.6875,
      "completions/min_length": 1053.0,
      "completions/min_terminated_length": 1053.0,
      "entropy": 0.05361162032932043,
      "epoch": 0.00576,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.6218546628952026,
      "kl": 0.02329419369925745,
      "learning_rate": 7.999969215498707e-06,
      "loss": -0.0147,
      "num_tokens": 5443189.0,
      "reward": 0.6181250214576721,
      "reward_std": 0.2755875587463379,
      "rewards/rollout_reward_func/mean": 0.6181250214576721,
      "rewards/rollout_reward_func/std": 0.45625001192092896,
      "sampling/importance_sampling_ratio/max": 1.4299272298812866,
      "sampling/importance_sampling_ratio/mean": 0.9309602975845337,
      "sampling/importance_sampling_ratio/min": 0.39184144139289856,
      "sampling/sampling_logp_difference/max": 0.9363220930099487,
      "sampling/sampling_logp_difference/mean": 0.011649301275610924,
      "step": 72,
      "step_time": 15.753308050000896
    },
    {
      "clip_ratio/high_max": 0.011660009622573853,
      "clip_ratio/high_mean": 0.005830004811286926,
      "clip_ratio/low_mean": 0.0017361111240461469,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.007566115935333073,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2801.0,
      "completions/max_terminated_length": 2801.0,
      "completions/mean_length": 1721.4375,
      "completions/mean_terminated_length": 1721.4375,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.06148386397399008,
      "epoch": 0.00584,
      "frac_reward_zero_std": 0.375,
      "grad_norm": 1.5323787927627563,
      "kl": 0.02491366575122811,
      "learning_rate": 7.999967481498294e-06,
      "loss": 0.0102,
      "num_tokens": 5509968.0,
      "reward": 0.5487499833106995,
      "reward_std": 0.33183753490448,
      "rewards/rollout_reward_func/mean": 0.5487499833106995,
      "rewards/rollout_reward_func/std": 0.4589731991291046,
      "sampling/importance_sampling_ratio/max": 1.9993343353271484,
      "sampling/importance_sampling_ratio/mean": 0.9622111916542053,
      "sampling/importance_sampling_ratio/min": 0.34618350863456726,
      "sampling/sampling_logp_difference/max": 1.054746150970459,
      "sampling/sampling_logp_difference/mean": 0.013135725632309914,
      "step": 73,
      "step_time": 16.009405589999915
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2792.0,
      "completions/max_terminated_length": 2792.0,
      "completions/mean_length": 1907.75,
      "completions/mean_terminated_length": 1907.75,
      "completions/min_length": 1052.0,
      "completions/min_terminated_length": 1052.0,
      "entropy": 0.08447153866291046,
      "epoch": 0.00592,
      "frac_reward_zero_std": 0.625,
      "grad_norm": 1.2818049192428589,
      "kl": 0.03082829067716375,
      "learning_rate": 7.999965699991369e-06,
      "loss": 0.0268,
      "num_tokens": 5583748.0,
      "reward": 0.44875001907348633,
      "reward_std": 0.20683756470680237,
      "rewards/rollout_reward_func/mean": 0.44875001907348633,
      "rewards/rollout_reward_func/std": 0.3857104480266571,
      "sampling/importance_sampling_ratio/max": 1.9656041860580444,
      "sampling/importance_sampling_ratio/mean": 1.0510060787200928,
      "sampling/importance_sampling_ratio/min": 0.4638214409351349,
      "sampling/sampling_logp_difference/max": 0.651539146900177,
      "sampling/sampling_logp_difference/mean": 0.016923408955335617,
      "step": 74,
      "step_time": 16.37548145899973
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0016447368543595076,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0016447368543595076,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 2794.0,
      "completions/max_terminated_length": 2794.0,
      "completions/mean_length": 1590.5,
      "completions/mean_terminated_length": 1590.5,
      "completions/min_length": 1054.0,
      "completions/min_terminated_length": 1054.0,
      "entropy": 0.051554064732044935,
      "epoch": 0.006,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.4656570851802826,
      "kl": 0.039722154615446925,
      "learning_rate": 7.99996387097796e-06,
      "loss": -0.0273,
      "num_tokens": 5646147.0,
      "reward": 0.8512499928474426,
      "reward_std": 0.13466876745224,
      "rewards/rollout_reward_func/mean": 0.8512499928474426,
      "rewards/rollout_reward_func/std": 0.4309011399745941,
      "sampling/importance_sampling_ratio/max": 2.459237575531006,
      "sampling/importance_sampling_ratio/mean": 1.0666892528533936,
      "sampling/importance_sampling_ratio/min": 0.403707355260849,
      "sampling/sampling_logp_difference/max": 0.9005355834960938,
      "sampling/sampling_logp_difference/mean": 0.012501123361289501,
      "step": 75,
      "step_time": 16.584246522000285
    },
    {
      "epoch": 0.006,
      "eval_clip_ratio/high_max": 0.0,
      "eval_clip_ratio/high_mean": 0.0,
      "eval_clip_ratio/low_mean": 0.0,
      "eval_clip_ratio/low_min": 0.0,
      "eval_clip_ratio/region_mean": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 2216.95,
      "eval_completions/max_terminated_length": 2216.95,
      "eval_completions/mean_length": 1925.65,
      "eval_completions/mean_terminated_length": 1925.65,
      "eval_completions/min_length": 1634.35,
      "eval_completions/min_terminated_length": 1634.35,
      "eval_entropy": 0.07575540114194154,
      "eval_frac_reward_zero_std": 1.0,
      "eval_kl": 0.031034281105894478,
      "eval_loss": 4.469734994927421e-05,
      "eval_num_tokens": 5646147.0,
      "eval_reward": 0.6004999987781048,
      "eval_reward_std": 0.0,
      "eval_rewards/rollout_reward_func/mean": 0.6004999987781048,
      "eval_rewards/rollout_reward_func/std": 0.24536602906882762,
      "eval_runtime": 13.987,
      "eval_samples_per_second": 0.715,
      "eval_sampling/importance_sampling_ratio/max": 1.2220476478338242,
      "eval_sampling/importance_sampling_ratio/mean": 1.0154326111078262,
      "eval_sampling/importance_sampling_ratio/min": 0.8088175728917122,
      "eval_sampling/sampling_logp_difference/max": 0.25984298419207336,
      "eval_sampling/sampling_logp_difference/mean": 0.014358489285223186,
      "eval_steps_per_second": 0.357,
      "step": 75
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 25000,
  "num_input_tokens_seen": 5646147,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}