{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.00075, "eval_steps": 500, "global_step": 75, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.0, "completions/max_terminated_length": 290.0, "completions/mean_length": 189.28125, "completions/mean_terminated_length": 189.28125, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "entropy": 6.437062740325928, "epoch": 1e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.016826510429382324, "kl": 0.0, "learning_rate": 0.0, "loss": 0.0013, "num_tokens": 19729.0, "reward": 0.3447500467300415, "reward_std": 0.442057728767395, "rewards/rollout_reward_func/mean": 0.3447500467300415, "rewards/rollout_reward_func/std": 0.5397446751594543, "sampling/importance_sampling_ratio/max": 0.09006787836551666, "sampling/importance_sampling_ratio/mean": 0.021625002846121788, "sampling/importance_sampling_ratio/min": 2.6737835799468434e-21, "sampling/sampling_logp_difference/max": 10.421490669250488, "sampling/sampling_logp_difference/mean": 0.9961081743240356, "step": 1, "step_time": 6.1302893520023645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.437062740325928, "epoch": 2e-05, "grad_norm": 0.01793765090405941, "kl": 0.0, "learning_rate": 2.8571428571428575e-07, "loss": 0.0013, "step": 2, "step_time": 2.0380342090029444 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.0, "completions/max_terminated_length": 311.0, "completions/mean_length": 181.75, "completions/mean_terminated_length": 181.75, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 6.339070081710815, "epoch": 3e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.01880541257560253, "kl": 0.0002252149279229343, "learning_rate": 5.714285714285715e-07, "loss": 0.0008, "num_tokens": 39137.0, "reward": 0.26824522018432617, "reward_std": 0.5006938576698303, "rewards/rollout_reward_func/mean": 0.26824522018432617, "rewards/rollout_reward_func/std": 0.6827228665351868, "sampling/importance_sampling_ratio/max": 0.08637334406375885, "sampling/importance_sampling_ratio/mean": 0.022774849086999893, "sampling/importance_sampling_ratio/min": 1.3291454737580292e-19, "sampling/sampling_logp_difference/max": 16.614532470703125, "sampling/sampling_logp_difference/mean": 0.9813731908798218, "step": 3, "step_time": 4.804124853002577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.339388012886047, "epoch": 4e-05, "grad_norm": 0.01893947273492813, "kl": 0.00022322818585962523, "learning_rate": 8.571428571428572e-07, "loss": 0.0008, "step": 4, "step_time": 2.015891280001597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 297.0, "completions/max_terminated_length": 297.0, "completions/mean_length": 180.5625, "completions/mean_terminated_length": 179.19354248046875, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 6.364191830158234, "epoch": 5e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.015179796144366264, "kl": 0.00024546185159124434, "learning_rate": 1.142857142857143e-06, "loss": -0.0013, "num_tokens": 58915.0, "reward": 0.30358171463012695, "reward_std": 0.5374526381492615, "rewards/rollout_reward_func/mean": 0.30358171463012695, "rewards/rollout_reward_func/std": 0.7121029496192932, "sampling/importance_sampling_ratio/max": 0.09015928953886032, "sampling/importance_sampling_ratio/mean": 0.028123818337917328, "sampling/importance_sampling_ratio/min": 6.7218247539707566e-18, "sampling/sampling_logp_difference/max": 3.8256945610046387, "sampling/sampling_logp_difference/mean": 0.9157212376594543, "step": 5, "step_time": 5.45849032499973 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.3711864948272705, "epoch": 6e-05, "grad_norm": 0.014959807507693768, "kl": 0.00025912884848366957, "learning_rate": 1.4285714285714286e-06, "loss": -0.0013, "step": 6, "step_time": 2.0148870949979028 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.0, "completions/max_terminated_length": 290.0, "completions/mean_length": 159.78125, "completions/mean_terminated_length": 159.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 6.184065341949463, "epoch": 7e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.020354198291897774, "kl": 0.00016448212227260228, "learning_rate": 1.7142857142857145e-06, "loss": 0.0001, "num_tokens": 77924.0, "reward": 0.4891502559185028, "reward_std": 0.6122604012489319, "rewards/rollout_reward_func/mean": 0.4891502559185028, "rewards/rollout_reward_func/std": 0.7439028024673462, "sampling/importance_sampling_ratio/max": 0.23023700714111328, "sampling/importance_sampling_ratio/mean": 0.04043475538492203, "sampling/importance_sampling_ratio/min": 0.0005593057139776647, "sampling/sampling_logp_difference/max": 1.5701570510864258, "sampling/sampling_logp_difference/mean": 0.7449289560317993, "step": 7, "step_time": 4.50355222700091 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.187336444854736, "epoch": 8e-05, "grad_norm": 0.02016047202050686, "kl": 0.00018859418923966587, "learning_rate": 2.0000000000000003e-06, "loss": 0.0001, "step": 8, "step_time": 2.0521075350043247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.0, "completions/max_terminated_length": 305.0, "completions/mean_length": 179.71875, "completions/mean_terminated_length": 179.71875, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "entropy": 6.317316174507141, "epoch": 9e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.011887814849615097, "kl": 0.0002640308539412217, "learning_rate": 2.285714285714286e-06, "loss": 0.0001, "num_tokens": 97755.0, "reward": 0.14386901259422302, "reward_std": 0.46610209345817566, "rewards/rollout_reward_func/mean": 0.14386901259422302, "rewards/rollout_reward_func/std": 0.6731327772140503, "sampling/importance_sampling_ratio/max": 0.07798832654953003, "sampling/importance_sampling_ratio/mean": 0.027132228016853333, "sampling/importance_sampling_ratio/min": 3.0175977556363997e-16, "sampling/sampling_logp_difference/max": 3.646486282348633, "sampling/sampling_logp_difference/mean": 0.8435187339782715, "step": 9, "step_time": 4.767648326000199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.318172335624695, "epoch": 0.0001, "grad_norm": 0.012068596668541431, "kl": 0.00021811862825416028, "learning_rate": 2.571428571428571e-06, "loss": 0.0001, "step": 10, "step_time": 3.0171621790013887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.0, "completions/max_terminated_length": 305.0, "completions/mean_length": 161.15625, "completions/mean_terminated_length": 161.15625, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 6.06863796710968, "epoch": 0.00011, "frac_reward_zero_std": 0.0, "grad_norm": 0.03053852915763855, "kl": 0.0002923815409303643, "learning_rate": 2.8571428571428573e-06, "loss": 0.0009, "num_tokens": 115960.0, "reward": 0.10083174705505371, "reward_std": 0.73536616563797, "rewards/rollout_reward_func/mean": 0.10083174705505371, "rewards/rollout_reward_func/std": 0.8196083307266235, "sampling/importance_sampling_ratio/max": 0.08715340495109558, "sampling/importance_sampling_ratio/mean": 0.039209067821502686, "sampling/importance_sampling_ratio/min": 0.0009637149632908404, "sampling/sampling_logp_difference/max": 1.487579345703125, "sampling/sampling_logp_difference/mean": 0.7100945711135864, "step": 11, "step_time": 4.602402659003928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.072819113731384, "epoch": 0.00012, "grad_norm": 0.030046585947275162, "kl": 0.00036826524046773557, "learning_rate": 3.142857142857143e-06, "loss": 0.0009, "step": 12, "step_time": 2.010943456001769 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 348.0, "completions/max_terminated_length": 348.0, "completions/mean_length": 176.90625, "completions/mean_terminated_length": 176.90625, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "entropy": 6.142941534519196, "epoch": 0.00013, "frac_reward_zero_std": 0.0, "grad_norm": 0.012423109263181686, "kl": 0.00031069645046954975, "learning_rate": 3.428571428571429e-06, "loss": 0.0, "num_tokens": 135893.0, "reward": 0.3551899194717407, "reward_std": 0.4100135564804077, "rewards/rollout_reward_func/mean": 0.3551899194717407, "rewards/rollout_reward_func/std": 0.4807114005088806, "sampling/importance_sampling_ratio/max": 0.08160404115915298, "sampling/importance_sampling_ratio/mean": 0.028775818645954132, "sampling/importance_sampling_ratio/min": 9.679094364400953e-05, "sampling/sampling_logp_difference/max": 1.6882461309432983, "sampling/sampling_logp_difference/mean": 0.7273038625717163, "step": 13, "step_time": 4.72503407199838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.149784088134766, "epoch": 0.00014, "grad_norm": 0.012431415729224682, "kl": 0.00031931480134517187, "learning_rate": 3.7142857142857146e-06, "loss": 0.0001, "step": 14, "step_time": 2.0601124099976005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 402.0, "completions/max_terminated_length": 402.0, "completions/mean_length": 185.1875, "completions/mean_terminated_length": 187.77418518066406, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 6.218902826309204, "epoch": 0.00015, "frac_reward_zero_std": 0.25, "grad_norm": 0.02713136561214924, "kl": 0.00027004735966329463, "learning_rate": 4.000000000000001e-06, "loss": 0.0015, "num_tokens": 155683.0, "reward": 0.305591344833374, "reward_std": 0.40960314869880676, "rewards/rollout_reward_func/mean": 0.305591344833374, "rewards/rollout_reward_func/std": 0.5242363810539246, "sampling/importance_sampling_ratio/max": 0.08599410951137543, "sampling/importance_sampling_ratio/mean": 0.025865234434604645, "sampling/importance_sampling_ratio/min": 7.349282722104652e-14, "sampling/sampling_logp_difference/max": 2.5293257236480713, "sampling/sampling_logp_difference/mean": 0.8104561567306519, "step": 15, "step_time": 4.914467659998991 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.2238587737083435, "epoch": 0.00016, "grad_norm": 0.026739850640296936, "kl": 0.00038530101119249593, "learning_rate": 4.2857142857142855e-06, "loss": 0.0015, "step": 16, "step_time": 2.6130522349994862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 346.0, "completions/max_terminated_length": 346.0, "completions/mean_length": 161.375, "completions/mean_terminated_length": 161.375, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "entropy": 6.213452756404877, "epoch": 0.00017, "frac_reward_zero_std": 0.0, "grad_norm": 0.03361396864056587, "kl": 0.0007407644588965923, "learning_rate": 4.571428571428572e-06, "loss": 0.0022, "num_tokens": 175471.0, "reward": 0.23945416510105133, "reward_std": 0.7443495988845825, "rewards/rollout_reward_func/mean": 0.23945416510105133, "rewards/rollout_reward_func/std": 0.8175519108772278, "sampling/importance_sampling_ratio/max": 0.0804901197552681, "sampling/importance_sampling_ratio/mean": 0.032769910991191864, "sampling/importance_sampling_ratio/min": 0.0004995061899535358, "sampling/sampling_logp_difference/max": 1.4847469329833984, "sampling/sampling_logp_difference/mean": 0.7405259609222412, "step": 17, "step_time": 4.827751421997164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.230209648609161, "epoch": 0.00018, "grad_norm": 0.03374752774834633, "kl": 0.001192268646263983, "learning_rate": 4.857142857142858e-06, "loss": 0.0021, "step": 18, "step_time": 2.0522313620022032 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 485.0, "completions/max_terminated_length": 485.0, "completions/mean_length": 174.4375, "completions/mean_terminated_length": 174.4375, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "entropy": 6.370483934879303, "epoch": 0.00019, "frac_reward_zero_std": 0.0, "grad_norm": 0.018522951751947403, "kl": 0.0011664420198940206, "learning_rate": 5.142857142857142e-06, "loss": -0.0016, "num_tokens": 195133.0, "reward": 0.25696271657943726, "reward_std": 0.5223400592803955, "rewards/rollout_reward_func/mean": 0.25696271657943726, "rewards/rollout_reward_func/std": 0.6738208532333374, "sampling/importance_sampling_ratio/max": 0.07579963654279709, "sampling/importance_sampling_ratio/mean": 0.0267631858587265, "sampling/importance_sampling_ratio/min": 3.3313353689413245e-22, "sampling/sampling_logp_difference/max": 4.017205715179443, "sampling/sampling_logp_difference/mean": 0.8683640956878662, "step": 19, "step_time": 5.342033982999055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.373503923416138, "epoch": 0.0002, "grad_norm": 0.018172789365053177, "kl": 0.0018378060049144551, "learning_rate": 5.428571428571429e-06, "loss": -0.0016, "step": 20, "step_time": 2.2543999580011587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.0, "completions/max_terminated_length": 260.0, "completions/mean_length": 141.34375, "completions/mean_terminated_length": 141.34375, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 6.3722938895225525, "epoch": 0.00021, "frac_reward_zero_std": 0.0, "grad_norm": 0.022796686738729477, "kl": 0.003324791803606786, "learning_rate": 5.7142857142857145e-06, "loss": -0.0005, "num_tokens": 213848.0, "reward": 0.23120194673538208, "reward_std": 0.46816450357437134, "rewards/rollout_reward_func/mean": 0.23120194673538208, "rewards/rollout_reward_func/std": 0.5511780977249146, "sampling/importance_sampling_ratio/max": 0.08035512268543243, "sampling/importance_sampling_ratio/mean": 0.03772790729999542, "sampling/importance_sampling_ratio/min": 9.345225998731152e-18, "sampling/sampling_logp_difference/max": 3.5961461067199707, "sampling/sampling_logp_difference/mean": 0.8969376683235168, "step": 21, "step_time": 4.52679192800133 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.381504118442535, "epoch": 0.00022, "grad_norm": 0.02249361015856266, "kl": 0.00530128157697618, "learning_rate": 6e-06, "loss": -0.0006, "step": 22, "step_time": 2.9375970069977484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 297.0, "completions/max_terminated_length": 297.0, "completions/mean_length": 131.78125, "completions/mean_terminated_length": 129.06451416015625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 6.41948789358139, "epoch": 0.00023, "frac_reward_zero_std": 0.0, "grad_norm": 0.02175409533083439, "kl": 0.008126991509925574, "learning_rate": 6.285714285714286e-06, "loss": 0.0015, "num_tokens": 232513.0, "reward": 0.27927568554878235, "reward_std": 0.5316575765609741, "rewards/rollout_reward_func/mean": 0.27927568554878235, "rewards/rollout_reward_func/std": 0.6082769632339478, "sampling/importance_sampling_ratio/max": 0.1689588725566864, "sampling/importance_sampling_ratio/mean": 0.04865079000592232, "sampling/importance_sampling_ratio/min": 8.09948256636731e-16, "sampling/sampling_logp_difference/max": 3.860705852508545, "sampling/sampling_logp_difference/mean": 0.8586986064910889, "step": 23, "step_time": 4.331727726001191 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.4265682101249695, "epoch": 0.00024, "grad_norm": 0.020918752998113632, "kl": 0.011424720054492354, "learning_rate": 6.571428571428572e-06, "loss": 0.0014, "step": 24, "step_time": 2.0076011149994883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 206.0, "completions/max_terminated_length": 206.0, "completions/mean_length": 107.78125, "completions/mean_terminated_length": 107.78125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 6.199184834957123, "epoch": 0.00025, "frac_reward_zero_std": 0.0, "grad_norm": 0.023662026971578598, "kl": 0.011069391970522702, "learning_rate": 6.857142857142858e-06, "loss": -0.0014, "num_tokens": 250218.0, "reward": 0.21139422059059143, "reward_std": 0.4235047399997711, "rewards/rollout_reward_func/mean": 0.21139422059059143, "rewards/rollout_reward_func/std": 0.5656381249427795, "sampling/importance_sampling_ratio/max": 0.14888890087604523, "sampling/importance_sampling_ratio/mean": 0.06519368290901184, "sampling/importance_sampling_ratio/min": 0.007011369336396456, "sampling/sampling_logp_difference/max": 1.5711028575897217, "sampling/sampling_logp_difference/mean": 0.7155030965805054, "step": 25, "step_time": 4.00336327399782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.216024577617645, "epoch": 0.00026, "grad_norm": 0.0240799430757761, "kl": 0.01640008925460279, "learning_rate": 7.1428571428571436e-06, "loss": -0.0015, "step": 26, "step_time": 1.988449569002114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 342.0, "completions/max_terminated_length": 342.0, "completions/mean_length": 181.625, "completions/mean_terminated_length": 181.625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 6.214400053024292, "epoch": 0.00027, "frac_reward_zero_std": 0.0, "grad_norm": 0.024456897750496864, "kl": 0.030537182930856943, "learning_rate": 7.428571428571429e-06, "loss": -0.0006, "num_tokens": 269790.0, "reward": 0.027159079909324646, "reward_std": 0.5774486064910889, "rewards/rollout_reward_func/mean": 0.027159079909324646, "rewards/rollout_reward_func/std": 0.5946774482727051, "sampling/importance_sampling_ratio/max": 0.12529020011425018, "sampling/importance_sampling_ratio/mean": 0.028741449117660522, "sampling/importance_sampling_ratio/min": 0.00032201220165006816, "sampling/sampling_logp_difference/max": 1.747678279876709, "sampling/sampling_logp_difference/mean": 0.7429477572441101, "step": 27, "step_time": 4.680500283000583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.226389408111572, "epoch": 0.00028, "grad_norm": 0.02442527562379837, "kl": 0.045675538247451186, "learning_rate": 7.714285714285716e-06, "loss": -0.0008, "step": 28, "step_time": 2.470751229000598 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.0, "completions/max_terminated_length": 282.0, "completions/mean_length": 162.40625, "completions/mean_terminated_length": 162.40625, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 6.057620167732239, "epoch": 0.00029, "frac_reward_zero_std": 0.0, "grad_norm": 0.016966896131634712, "kl": 0.04296685662120581, "learning_rate": 8.000000000000001e-06, "loss": -0.0001, "num_tokens": 288771.0, "reward": 0.36989423632621765, "reward_std": 0.14534157514572144, "rewards/rollout_reward_func/mean": 0.36989423632621765, "rewards/rollout_reward_func/std": 0.27579429745674133, "sampling/importance_sampling_ratio/max": 0.121132493019104, "sampling/importance_sampling_ratio/mean": 0.03923218697309494, "sampling/importance_sampling_ratio/min": 0.0010416348231956363, "sampling/sampling_logp_difference/max": 1.7733206748962402, "sampling/sampling_logp_difference/mean": 0.7076070308685303, "step": 29, "step_time": 4.446439086999817 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.042197942733765, "epoch": 0.0003, "grad_norm": 0.017070813104510307, "kl": 0.057768960017710924, "learning_rate": 8.285714285714287e-06, "loss": -0.0, "step": 30, "step_time": 2.0213948890032043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.0, "completions/max_terminated_length": 338.0, "completions/mean_length": 170.75, "completions/mean_terminated_length": 170.75, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "entropy": 6.304847180843353, "epoch": 0.00031, "frac_reward_zero_std": 0.0, "grad_norm": 0.02980108931660652, "kl": 0.0634244168177247, "learning_rate": 8.571428571428571e-06, "loss": -0.003, "num_tokens": 307723.0, "reward": 0.39828288555145264, "reward_std": 0.20638835430145264, "rewards/rollout_reward_func/mean": 0.39828288555145264, "rewards/rollout_reward_func/std": 0.3757359981536865, "sampling/importance_sampling_ratio/max": 0.1449124813079834, "sampling/importance_sampling_ratio/mean": 0.04636143520474434, "sampling/importance_sampling_ratio/min": 1.888638362217978e-16, "sampling/sampling_logp_difference/max": 3.443108558654785, "sampling/sampling_logp_difference/mean": 0.8668484091758728, "step": 31, "step_time": 4.895437238999875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 6.228823721408844, "epoch": 0.00032, "grad_norm": 0.031759388744831085, "kl": 0.08063395507633686, "learning_rate": 8.857142857142858e-06, "loss": -0.0031, "step": 32, "step_time": 2.0456357880011637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.0, "completions/max_terminated_length": 274.0, "completions/mean_length": 154.9375, "completions/mean_terminated_length": 154.9375, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "entropy": 5.898605585098267, "epoch": 0.00033, "frac_reward_zero_std": 0.0, "grad_norm": 0.02342640608549118, "kl": 0.09808124136179686, "learning_rate": 9.142857142857144e-06, "loss": -0.0015, "num_tokens": 327073.0, "reward": 0.4524267315864563, "reward_std": 0.3188154101371765, "rewards/rollout_reward_func/mean": 0.4524267315864563, "rewards/rollout_reward_func/std": 0.42413368821144104, "sampling/importance_sampling_ratio/max": 0.1602727472782135, "sampling/importance_sampling_ratio/mean": 0.06595402956008911, "sampling/importance_sampling_ratio/min": 1.780031055886866e-07, "sampling/sampling_logp_difference/max": 2.5631158351898193, "sampling/sampling_logp_difference/mean": 0.6789650917053223, "step": 33, "step_time": 5.394653027000459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 5.718324422836304, "epoch": 0.00034, "grad_norm": 0.024896224960684776, "kl": 0.11854369938373566, "learning_rate": 9.42857142857143e-06, "loss": -0.0016, "step": 34, "step_time": 2.0065684620003594 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.0, "completions/max_terminated_length": 298.0, "completions/mean_length": 188.15625, "completions/mean_terminated_length": 188.15625, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "entropy": 5.479340612888336, "epoch": 0.00035, "frac_reward_zero_std": 0.0, "grad_norm": 0.05328498035669327, "kl": 0.17625079490244389, "learning_rate": 9.714285714285715e-06, "loss": 0.0055, "num_tokens": 347582.0, "reward": 0.6540457010269165, "reward_std": 0.3883731961250305, "rewards/rollout_reward_func/mean": 0.6540457010269165, "rewards/rollout_reward_func/std": 0.5172057151794434, "sampling/importance_sampling_ratio/max": 0.28821438550949097, "sampling/importance_sampling_ratio/mean": 0.05695977061986923, "sampling/importance_sampling_ratio/min": 2.9049160646388295e-11, "sampling/sampling_logp_difference/max": 18.75899314880371, "sampling/sampling_logp_difference/mean": 0.7121621966362, "step": 35, "step_time": 4.564382323002064 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 5.390263617038727, "epoch": 0.00036, "grad_norm": 0.053973328322172165, "kl": 0.1723556686192751, "learning_rate": 1e-05, "loss": 0.0055, "step": 36, "step_time": 2.0383502150034474 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.0, "completions/max_terminated_length": 305.0, "completions/mean_length": 144.96875, "completions/mean_terminated_length": 144.96875, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "entropy": 5.320340633392334, "epoch": 0.00037, "frac_reward_zero_std": 0.0, "grad_norm": 0.03199679031968117, "kl": 0.12412955984473228, "learning_rate": 9.9999999995372e-06, "loss": 0.0032, "num_tokens": 366173.0, "reward": 0.6232043504714966, "reward_std": 0.2902255356311798, "rewards/rollout_reward_func/mean": 0.6232043504714966, "rewards/rollout_reward_func/std": 0.4259689748287201, "sampling/importance_sampling_ratio/max": 0.2735931873321533, "sampling/importance_sampling_ratio/mean": 0.11970683187246323, "sampling/importance_sampling_ratio/min": 1.908677859319141e-06, "sampling/sampling_logp_difference/max": 3.270216703414917, "sampling/sampling_logp_difference/mean": 0.5641312599182129, "step": 37, "step_time": 4.327299739001319 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "entropy": 5.401705622673035, "epoch": 0.00038, "grad_norm": 0.030774196609854698, "kl": 0.10921481531113386, "learning_rate": 9.999999998148802e-06, "loss": 0.0031, "step": 38, "step_time": 2.0147021009979653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.0, "completions/max_terminated_length": 260.0, "completions/mean_length": 141.03125, "completions/mean_terminated_length": 141.03125, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 5.776544392108917, "epoch": 0.00039, "frac_reward_zero_std": 0.0, "grad_norm": 0.023018015548586845, "kl": 0.10279421554878354, "learning_rate": 9.999999995834804e-06, "loss": -0.0019, "num_tokens": 384990.0, "reward": 0.5750877857208252, "reward_std": 0.20346705615520477, "rewards/rollout_reward_func/mean": 0.5750877857208252, "rewards/rollout_reward_func/std": 0.4033717215061188, "sampling/importance_sampling_ratio/max": 0.2024683803319931, "sampling/importance_sampling_ratio/mean": 0.07642973214387894, "sampling/importance_sampling_ratio/min": 1.974384267189322e-10, "sampling/sampling_logp_difference/max": 3.7110068798065186, "sampling/sampling_logp_difference/mean": 0.6953427791595459, "step": 39, "step_time": 4.782118793002155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 5.8619155287742615, "epoch": 0.0004, "grad_norm": 0.02198023535311222, "kl": 0.09226133674383163, "learning_rate": 9.999999992595207e-06, "loss": -0.002, "step": 40, "step_time": 2.003476854000837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 360.0, "completions/max_terminated_length": 360.0, "completions/mean_length": 176.875, "completions/mean_terminated_length": 176.875, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "entropy": 5.619787395000458, "epoch": 0.00041, "frac_reward_zero_std": 0.25, "grad_norm": 0.02750505320727825, "kl": 0.07494280813261867, "learning_rate": 9.999999988430008e-06, "loss": -0.0014, "num_tokens": 404818.0, "reward": 0.632590115070343, "reward_std": 0.12197521328926086, "rewards/rollout_reward_func/mean": 0.632590115070343, "rewards/rollout_reward_func/std": 0.3571937084197998, "sampling/importance_sampling_ratio/max": 0.18347331881523132, "sampling/importance_sampling_ratio/mean": 0.06567263603210449, "sampling/importance_sampling_ratio/min": 0.0007156870560720563, "sampling/sampling_logp_difference/max": 1.7460063695907593, "sampling/sampling_logp_difference/mean": 0.5835450887680054, "step": 41, "step_time": 4.588236302999576 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 5.655375897884369, "epoch": 0.00042, "grad_norm": 0.028079615905880928, "kl": 0.07384081510826945, "learning_rate": 9.999999983339212e-06, "loss": -0.0015, "step": 42, "step_time": 2.058184131998132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.0, "completions/max_terminated_length": 338.0, "completions/mean_length": 147.3125, "completions/mean_terminated_length": 147.3125, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 5.985348284244537, "epoch": 0.00043, "frac_reward_zero_std": 0.0, "grad_norm": 0.04133318364620209, "kl": 0.07979621039703488, "learning_rate": 9.999999977322818e-06, "loss": -0.0052, "num_tokens": 423108.0, "reward": 0.39953890442848206, "reward_std": 0.2123834490776062, "rewards/rollout_reward_func/mean": 0.39953890442848206, "rewards/rollout_reward_func/std": 0.2874828279018402, "sampling/importance_sampling_ratio/max": 0.1767341047525406, "sampling/importance_sampling_ratio/mean": 0.06585769355297089, "sampling/importance_sampling_ratio/min": 9.253626085648562e-19, "sampling/sampling_logp_difference/max": 3.958195209503174, "sampling/sampling_logp_difference/mean": 0.775503933429718, "step": 43, "step_time": 4.575371188000645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 5.950318694114685, "epoch": 0.00044, "grad_norm": 0.03985368832945824, "kl": 0.08474360965192318, "learning_rate": 9.999999970380822e-06, "loss": -0.0053, "step": 44, "step_time": 2.516747978001149 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 369.0, "completions/max_terminated_length": 369.0, "completions/mean_length": 193.75, "completions/mean_terminated_length": 193.75, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "entropy": 5.851812183856964, "epoch": 0.00045, "frac_reward_zero_std": 0.0, "grad_norm": 0.03772180527448654, "kl": 0.10888817999511957, "learning_rate": 9.999999962513228e-06, "loss": -0.006, "num_tokens": 442876.0, "reward": 0.6410436630249023, "reward_std": 0.7430479526519775, "rewards/rollout_reward_func/mean": 0.6410436630249023, "rewards/rollout_reward_func/std": 0.8171493411064148, "sampling/importance_sampling_ratio/max": 0.22074148058891296, "sampling/importance_sampling_ratio/mean": 0.05440264195203781, "sampling/importance_sampling_ratio/min": 7.153796750537485e-16, "sampling/sampling_logp_difference/max": 4.365869522094727, "sampling/sampling_logp_difference/mean": 0.780392050743103, "step": 45, "step_time": 5.305774641001335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 5.684995532035828, "epoch": 0.00046, "grad_norm": 0.039422787725925446, "kl": 0.12296807020902634, "learning_rate": 9.999999953720035e-06, "loss": -0.0063, "step": 46, "step_time": 2.0743733209983475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 348.0, "completions/max_terminated_length": 348.0, "completions/mean_length": 198.78125, "completions/mean_terminated_length": 198.78125, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "entropy": 5.193369805812836, "epoch": 0.00047, "frac_reward_zero_std": 0.0, "grad_norm": 0.03668779879808426, "kl": 0.12609337456524372, "learning_rate": 9.99999994400124e-06, "loss": -0.0051, "num_tokens": 463485.0, "reward": 0.2012612223625183, "reward_std": 0.6530838012695312, "rewards/rollout_reward_func/mean": 0.2012612223625183, "rewards/rollout_reward_func/std": 0.7242118716239929, "sampling/importance_sampling_ratio/max": 0.31475865840911865, "sampling/importance_sampling_ratio/mean": 0.05936652421951294, "sampling/importance_sampling_ratio/min": 0.00022013229317963123, "sampling/sampling_logp_difference/max": 1.833187460899353, "sampling/sampling_logp_difference/mean": 0.5444918870925903, "step": 47, "step_time": 4.7334515290003765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 4.908747911453247, "epoch": 0.00048, "grad_norm": 0.03906744718551636, "kl": 0.14741076529026031, "learning_rate": 9.999999933356848e-06, "loss": -0.0055, "step": 48, "step_time": 2.059140684999875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.0, "completions/max_terminated_length": 290.0, "completions/mean_length": 153.5625, "completions/mean_terminated_length": 153.5625, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 4.577030658721924, "epoch": 0.00049, "frac_reward_zero_std": 0.25, "grad_norm": 0.03131258487701416, "kl": 0.18377518095076084, "learning_rate": 9.999999921786855e-06, "loss": 0.002, "num_tokens": 482591.0, "reward": 0.41418272256851196, "reward_std": 0.16014696657657623, "rewards/rollout_reward_func/mean": 0.41418272256851196, "rewards/rollout_reward_func/std": 0.3027903735637665, "sampling/importance_sampling_ratio/max": 0.39005929231643677, "sampling/importance_sampling_ratio/mean": 0.15701675415039062, "sampling/importance_sampling_ratio/min": 0.007550857961177826, "sampling/sampling_logp_difference/max": 1.3340117931365967, "sampling/sampling_logp_difference/mean": 0.44514331221580505, "step": 49, "step_time": 4.443075770999712 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 4.31759974360466, "epoch": 0.0005, "grad_norm": 0.03223428502678871, "kl": 0.1999241877347231, "learning_rate": 9.999999909291265e-06, "loss": 0.0021, "step": 50, "step_time": 2.9629076719993463 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 205.0, "completions/max_terminated_length": 205.0, "completions/mean_length": 118.625, "completions/mean_terminated_length": 115.83870697021484, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 4.087187260389328, "epoch": 0.00051, "frac_reward_zero_std": 0.0, "grad_norm": 0.07074335217475891, "kl": 0.30883508175611496, "learning_rate": 9.999999895870075e-06, "loss": -0.0001, "num_tokens": 500235.0, "reward": 0.6331731081008911, "reward_std": 0.15591080486774445, "rewards/rollout_reward_func/mean": 0.6331731081008911, "rewards/rollout_reward_func/std": 0.26999494433403015, "sampling/importance_sampling_ratio/max": 0.5052566528320312, "sampling/importance_sampling_ratio/mean": 0.21748122572898865, "sampling/importance_sampling_ratio/min": 0.01564314402639866, "sampling/sampling_logp_difference/max": 1.5473060607910156, "sampling/sampling_logp_difference/mean": 0.4317389726638794, "step": 51, "step_time": 4.16781128099683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 3.906628429889679, "epoch": 0.00052, "grad_norm": 0.07227003574371338, "kl": 0.300492724403739, "learning_rate": 9.999999881523285e-06, "loss": -0.0003, "step": 52, "step_time": 1.9936077560014382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.0, "completions/max_terminated_length": 286.0, "completions/mean_length": 151.84375, "completions/mean_terminated_length": 151.84375, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "entropy": 3.3781526386737823, "epoch": 0.00053, "frac_reward_zero_std": 0.5, "grad_norm": 0.13030877709388733, "kl": 0.2350521795451641, "learning_rate": 9.999999866250896e-06, "loss": 0.012, "num_tokens": 519286.0, "reward": 0.6449999809265137, "reward_std": 0.11053535342216492, "rewards/rollout_reward_func/mean": 0.6449999809265137, "rewards/rollout_reward_func/std": 0.22530117630958557, "sampling/importance_sampling_ratio/max": 0.5501136779785156, "sampling/importance_sampling_ratio/mean": 0.2970014214515686, "sampling/importance_sampling_ratio/min": 0.029202939942479134, "sampling/sampling_logp_difference/max": 0.9902606010437012, "sampling/sampling_logp_difference/mean": 0.2762574255466461, "step": 53, "step_time": 4.454380056002265 }, { "clip_ratio/high_max": 0.028125000186264515, "clip_ratio/high_mean": 0.014062500093132257, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014062500093132257, "entropy": 3.4167210161685944, "epoch": 0.00054, "grad_norm": 0.11076293885707855, "kl": 0.22031726501882076, "learning_rate": 9.999999850052909e-06, "loss": 0.0108, "step": 54, "step_time": 2.026400898999782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 211.0, "completions/max_terminated_length": 197.0, "completions/mean_length": 155.5, "completions/mean_terminated_length": 155.43333435058594, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "entropy": 3.86805123090744, "epoch": 0.00055, "frac_reward_zero_std": 0.0, "grad_norm": 0.05002529174089432, "kl": 0.17954419739544392, "learning_rate": 9.99999983292932e-06, "loss": -0.0047, "num_tokens": 538070.0, "reward": 0.49569714069366455, "reward_std": 0.242295503616333, "rewards/rollout_reward_func/mean": 0.49569714069366455, "rewards/rollout_reward_func/std": 0.35909703373908997, "sampling/importance_sampling_ratio/max": 0.4999794065952301, "sampling/importance_sampling_ratio/mean": 0.22611427307128906, "sampling/importance_sampling_ratio/min": 2.373902644344905e-17, "sampling/sampling_logp_difference/max": 3.7852988243103027, "sampling/sampling_logp_difference/mean": 0.4584958851337433, "step": 55, "step_time": 4.410495514002832 }, { "clip_ratio/high_max": 0.010416666977107525, "clip_ratio/high_mean": 0.0052083334885537624, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "entropy": 3.928067147731781, "epoch": 0.00056, "grad_norm": 0.047599323093891144, "kl": 0.16341876238584518, "learning_rate": 9.999999814880132e-06, "loss": -0.0052, "step": 56, "step_time": 2.9177981660013756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 342.0, "completions/max_terminated_length": 342.0, "completions/mean_length": 167.96875, "completions/mean_terminated_length": 169.96774291992188, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "entropy": 3.8915408849716187, "epoch": 0.00057, "frac_reward_zero_std": 0.0, "grad_norm": 0.0617266409099102, "kl": 0.16402203310281038, "learning_rate": 9.999999795905347e-06, "loss": -0.0137, "num_tokens": 556741.0, "reward": 0.2770588994026184, "reward_std": 0.5140299201011658, "rewards/rollout_reward_func/mean": 0.2770588994026184, "rewards/rollout_reward_func/std": 0.6300316452980042, "sampling/importance_sampling_ratio/max": 0.4707293212413788, "sampling/importance_sampling_ratio/mean": 0.20163728296756744, "sampling/importance_sampling_ratio/min": 0.009354846552014351, "sampling/sampling_logp_difference/max": 1.20766019821167, "sampling/sampling_logp_difference/mean": 0.3649933934211731, "step": 57, "step_time": 4.600740758000029 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 3.8189143240451813, "epoch": 0.00058, "grad_norm": 0.060559939593076706, "kl": 0.15883306972682476, "learning_rate": 9.999999776004962e-06, "loss": -0.0137, "step": 58, "step_time": 2.0522955199994612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.0, "completions/max_terminated_length": 297.0, "completions/mean_length": 142.46875, "completions/mean_terminated_length": 142.46875, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "entropy": 3.5833038985729218, "epoch": 0.00059, "frac_reward_zero_std": 0.0, "grad_norm": 0.0712868869304657, "kl": 0.1435947585850954, "learning_rate": 9.999999755178978e-06, "loss": -0.0083, "num_tokens": 575084.0, "reward": 0.35289904475212097, "reward_std": 0.3668435215950012, "rewards/rollout_reward_func/mean": 0.35289904475212097, "rewards/rollout_reward_func/std": 0.5060283541679382, "sampling/importance_sampling_ratio/max": 0.449856698513031, "sampling/importance_sampling_ratio/mean": 0.27609291672706604, "sampling/importance_sampling_ratio/min": 0.03381441906094551, "sampling/sampling_logp_difference/max": 0.9775153398513794, "sampling/sampling_logp_difference/mean": 0.3117252588272095, "step": 59, "step_time": 4.590389930997844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 3.4226994812488556, "epoch": 0.0006, "grad_norm": 0.07168518751859665, "kl": 0.1514890482649207, "learning_rate": 9.999999733427394e-06, "loss": -0.0084, "step": 60, "step_time": 2.024904317002438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 206.0, "completions/max_terminated_length": 206.0, "completions/mean_length": 145.34375, "completions/mean_terminated_length": 145.34375, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 3.281781107187271, "epoch": 0.00061, "frac_reward_zero_std": 0.0, "grad_norm": 0.05610927194356918, "kl": 0.18225542083382607, "learning_rate": 9.99999971075021e-06, "loss": 0.0011, "num_tokens": 593311.0, "reward": 0.57869952917099, "reward_std": 0.34909936785697937, "rewards/rollout_reward_func/mean": 0.57869952917099, "rewards/rollout_reward_func/std": 0.538192093372345, "sampling/importance_sampling_ratio/max": 0.585938572883606, "sampling/importance_sampling_ratio/mean": 0.2803860306739807, "sampling/importance_sampling_ratio/min": 0.05912167578935623, "sampling/sampling_logp_difference/max": 0.9743081331253052, "sampling/sampling_logp_difference/mean": 0.2922592759132385, "step": 61, "step_time": 4.679663689001245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 3.1316112875938416, "epoch": 0.00062, "grad_norm": 0.059742532670497894, "kl": 0.19533881545066833, "learning_rate": 9.999999687147426e-06, "loss": 0.0009, "step": 62, "step_time": 2.4762866889977886 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.0, "completions/max_terminated_length": 258.0, "completions/mean_length": 137.0, "completions/mean_terminated_length": 137.0, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 3.003155827522278, "epoch": 0.00063, "frac_reward_zero_std": 0.0, "grad_norm": 0.11588791012763977, "kl": 0.19905831664800644, "learning_rate": 9.999999662619046e-06, "loss": -0.0209, "num_tokens": 612247.0, "reward": 0.4884374737739563, "reward_std": 0.4819467067718506, "rewards/rollout_reward_func/mean": 0.4884374737739563, "rewards/rollout_reward_func/std": 0.5825604200363159, "sampling/importance_sampling_ratio/max": 0.659873366355896, "sampling/importance_sampling_ratio/mean": 0.3772771954536438, "sampling/importance_sampling_ratio/min": 0.020119559019804, "sampling/sampling_logp_difference/max": 1.0678739547729492, "sampling/sampling_logp_difference/mean": 0.25328272581100464, "step": 63, "step_time": 4.334251456000857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "entropy": 2.8085409104824066, "epoch": 0.00064, "grad_norm": 0.11190968751907349, "kl": 0.2239690087735653, "learning_rate": 9.999999637165062e-06, "loss": -0.022, "step": 64, "step_time": 2.00917297399792 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.0, "completions/max_terminated_length": 258.0, "completions/mean_length": 160.6875, "completions/mean_terminated_length": 160.6875, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 2.5678041726350784, "epoch": 0.00065, "frac_reward_zero_std": 0.0, "grad_norm": 0.09978003054857254, "kl": 0.235586229711771, "learning_rate": 9.999999610785483e-06, "loss": 0.0039, "num_tokens": 631261.0, "reward": 0.5577644109725952, "reward_std": 0.28208568692207336, "rewards/rollout_reward_func/mean": 0.5577644109725952, "rewards/rollout_reward_func/std": 0.4251752495765686, "sampling/importance_sampling_ratio/max": 0.6934138536453247, "sampling/importance_sampling_ratio/mean": 0.3842087984085083, "sampling/importance_sampling_ratio/min": 0.05158117786049843, "sampling/sampling_logp_difference/max": 1.5275554656982422, "sampling/sampling_logp_difference/mean": 0.22714273631572723, "step": 65, "step_time": 4.462041075998059 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "entropy": 2.4012295603752136, "epoch": 0.00066, "grad_norm": 0.10902947187423706, "kl": 0.266902232542634, "learning_rate": 9.999999583480304e-06, "loss": 0.004, "step": 66, "step_time": 1.9998026810044394 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.0, "completions/max_terminated_length": 338.0, "completions/mean_length": 183.125, "completions/mean_terminated_length": 183.125, "completions/min_length": 91.0, "completions/min_terminated_length": 91.0, "entropy": 2.563511222600937, "epoch": 0.00067, "frac_reward_zero_std": 0.0, "grad_norm": 0.09899244457483292, "kl": 0.21912122517824173, "learning_rate": 9.999999555249524e-06, "loss": 0.0031, "num_tokens": 651201.0, "reward": 0.346504807472229, "reward_std": 0.37360942363739014, "rewards/rollout_reward_func/mean": 0.346504807472229, "rewards/rollout_reward_func/std": 0.47065863013267517, "sampling/importance_sampling_ratio/max": 0.5819504857063293, "sampling/importance_sampling_ratio/mean": 0.35113734006881714, "sampling/importance_sampling_ratio/min": 0.023396655917167664, "sampling/sampling_logp_difference/max": 0.822979211807251, "sampling/sampling_logp_difference/mean": 0.21178027987480164, "step": 67, "step_time": 4.976440294996792 }, { "clip_ratio/high_max": 0.008928571827709675, "clip_ratio/high_mean": 0.004464285913854837, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "entropy": 2.4892768561840057, "epoch": 0.00068, "grad_norm": 0.0969628244638443, "kl": 0.2153633050620556, "learning_rate": 9.999999526093148e-06, "loss": 0.0029, "step": 68, "step_time": 2.5076440479988378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 350.0, "completions/max_terminated_length": 350.0, "completions/mean_length": 184.375, "completions/mean_terminated_length": 184.375, "completions/min_length": 91.0, "completions/min_terminated_length": 91.0, "entropy": 2.2106962502002716, "epoch": 0.00069, "frac_reward_zero_std": 0.0, "grad_norm": 0.16426467895507812, "kl": 0.3220958895981312, "learning_rate": 9.999999496011169e-06, "loss": -0.0045, "num_tokens": 671269.0, "reward": 0.7414458990097046, "reward_std": 0.2716183662414551, "rewards/rollout_reward_func/mean": 0.7414458990097046, "rewards/rollout_reward_func/std": 0.4371891915798187, "sampling/importance_sampling_ratio/max": 0.7719125747680664, "sampling/importance_sampling_ratio/mean": 0.45400264859199524, "sampling/importance_sampling_ratio/min": 0.031057482585310936, "sampling/sampling_logp_difference/max": 1.5289125442504883, "sampling/sampling_logp_difference/mean": 0.18032191693782806, "step": 69, "step_time": 4.6545613879952725 }, { "clip_ratio/high_max": 0.030357143841683865, "clip_ratio/high_mean": 0.015178571920841932, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01964285783469677, "entropy": 2.1326727122068405, "epoch": 0.0007, "grad_norm": 0.12904977798461914, "kl": 0.3180929869413376, "learning_rate": 9.999999465003593e-06, "loss": -0.0057, "step": 70, "step_time": 2.0704947099984565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 175.84375, "completions/mean_terminated_length": 175.84375, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 2.0988260358572006, "epoch": 0.00071, "frac_reward_zero_std": 0.0, "grad_norm": 0.3079679012298584, "kl": 0.2244853936135769, "learning_rate": 9.999999433070417e-06, "loss": -0.0203, "num_tokens": 691168.0, "reward": 0.5884735584259033, "reward_std": 0.47160544991493225, "rewards/rollout_reward_func/mean": 0.5884735584259033, "rewards/rollout_reward_func/std": 0.6704621911048889, "sampling/importance_sampling_ratio/max": 0.638032853603363, "sampling/importance_sampling_ratio/mean": 0.4377201199531555, "sampling/importance_sampling_ratio/min": 0.1587381660938263, "sampling/sampling_logp_difference/max": 0.907859206199646, "sampling/sampling_logp_difference/mean": 0.15975390374660492, "step": 71, "step_time": 4.186460018994694 }, { "clip_ratio/high_max": 0.07083333469927311, "clip_ratio/high_mean": 0.05208333441987634, "clip_ratio/low_mean": 0.04062500083819032, "clip_ratio/low_min": 0.02083333395421505, "clip_ratio/region_mean": 0.09270833572372794, "entropy": 1.9240455776453018, "epoch": 0.00072, "grad_norm": 0.22433313727378845, "kl": 0.23141289688646793, "learning_rate": 9.999999400211643e-06, "loss": -0.0249, "step": 72, "step_time": 1.9681833920003555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.0, "completions/max_terminated_length": 262.0, "completions/mean_length": 168.46875, "completions/mean_terminated_length": 168.46875, "completions/min_length": 97.0, "completions/min_terminated_length": 97.0, "entropy": 1.7944766581058502, "epoch": 0.00073, "frac_reward_zero_std": 0.0, "grad_norm": 0.3287738561630249, "kl": 0.27973234094679356, "learning_rate": 9.99999936642727e-06, "loss": -0.0086, "num_tokens": 711023.0, "reward": 0.5628845691680908, "reward_std": 0.23407763242721558, "rewards/rollout_reward_func/mean": 0.5628845691680908, "rewards/rollout_reward_func/std": 0.41712766885757446, "sampling/importance_sampling_ratio/max": 0.8928364515304565, "sampling/importance_sampling_ratio/mean": 0.5339679718017578, "sampling/importance_sampling_ratio/min": 0.031287215650081635, "sampling/sampling_logp_difference/max": 2.0212931632995605, "sampling/sampling_logp_difference/mean": 0.14184574782848358, "step": 73, "step_time": 5.250511515001563 }, { "clip_ratio/high_max": 0.07500000111758709, "clip_ratio/high_mean": 0.054166668094694614, "clip_ratio/low_mean": 0.11197916977107525, "clip_ratio/low_min": 0.056250001303851604, "clip_ratio/region_mean": 0.16614583879709244, "entropy": 1.5351449102163315, "epoch": 0.00074, "grad_norm": 0.1667291224002838, "kl": 0.3220853917300701, "learning_rate": 9.999999331717294e-06, "loss": -0.0118, "step": 74, "step_time": 1.9973168569995323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.0, "completions/max_terminated_length": 305.0, "completions/mean_length": 187.75, "completions/mean_terminated_length": 187.75, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "entropy": 1.4904573261737823, "epoch": 0.00075, "frac_reward_zero_std": 0.0, "grad_norm": 0.34346434473991394, "kl": 0.2825224921107292, "learning_rate": 9.999999296081722e-06, "loss": 0.035, "num_tokens": 730815.0, "reward": 0.3780096173286438, "reward_std": 0.4338669776916504, "rewards/rollout_reward_func/mean": 0.3780096173286438, "rewards/rollout_reward_func/std": 0.6683818101882935, "sampling/importance_sampling_ratio/max": 1.004560947418213, "sampling/importance_sampling_ratio/mean": 0.5335597991943359, "sampling/importance_sampling_ratio/min": 9.286272306674448e-10, "sampling/sampling_logp_difference/max": 18.32341766357422, "sampling/sampling_logp_difference/mean": 0.260251522064209, "step": 75, "step_time": 4.602757915003167 } ], "logging_steps": 1.0, "max_steps": 200000, "num_input_tokens_seen": 730815, "num_train_epochs": 2, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }