| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.00075, |
| "eval_steps": 500, |
| "global_step": 75, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 290.0, |
| "completions/max_terminated_length": 290.0, |
| "completions/mean_length": 189.28125, |
| "completions/mean_terminated_length": 189.28125, |
| "completions/min_length": 87.0, |
| "completions/min_terminated_length": 87.0, |
| "entropy": 6.437062740325928, |
| "epoch": 1e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.016826510429382324, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": 0.0013, |
| "num_tokens": 19729.0, |
| "reward": 0.3447500467300415, |
| "reward_std": 0.442057728767395, |
| "rewards/rollout_reward_func/mean": 0.3447500467300415, |
| "rewards/rollout_reward_func/std": 0.5397446751594543, |
| "sampling/importance_sampling_ratio/max": 0.09006787836551666, |
| "sampling/importance_sampling_ratio/mean": 0.021625002846121788, |
| "sampling/importance_sampling_ratio/min": 2.6737835799468434e-21, |
| "sampling/sampling_logp_difference/max": 10.421490669250488, |
| "sampling/sampling_logp_difference/mean": 0.9961081743240356, |
| "step": 1, |
| "step_time": 6.1302893520023645 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.437062740325928, |
| "epoch": 2e-05, |
| "grad_norm": 0.01793765090405941, |
| "kl": 0.0, |
| "learning_rate": 2.8571428571428575e-07, |
| "loss": 0.0013, |
| "step": 2, |
| "step_time": 2.0380342090029444 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 311.0, |
| "completions/max_terminated_length": 311.0, |
| "completions/mean_length": 181.75, |
| "completions/mean_terminated_length": 181.75, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 6.339070081710815, |
| "epoch": 3e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01880541257560253, |
| "kl": 0.0002252149279229343, |
| "learning_rate": 5.714285714285715e-07, |
| "loss": 0.0008, |
| "num_tokens": 39137.0, |
| "reward": 0.26824522018432617, |
| "reward_std": 0.5006938576698303, |
| "rewards/rollout_reward_func/mean": 0.26824522018432617, |
| "rewards/rollout_reward_func/std": 0.6827228665351868, |
| "sampling/importance_sampling_ratio/max": 0.08637334406375885, |
| "sampling/importance_sampling_ratio/mean": 0.022774849086999893, |
| "sampling/importance_sampling_ratio/min": 1.3291454737580292e-19, |
| "sampling/sampling_logp_difference/max": 16.614532470703125, |
| "sampling/sampling_logp_difference/mean": 0.9813731908798218, |
| "step": 3, |
| "step_time": 4.804124853002577 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.339388012886047, |
| "epoch": 4e-05, |
| "grad_norm": 0.01893947273492813, |
| "kl": 0.00022322818585962523, |
| "learning_rate": 8.571428571428572e-07, |
| "loss": 0.0008, |
| "step": 4, |
| "step_time": 2.015891280001597 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 297.0, |
| "completions/max_terminated_length": 297.0, |
| "completions/mean_length": 180.5625, |
| "completions/mean_terminated_length": 179.19354248046875, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 6.364191830158234, |
| "epoch": 5e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.015179796144366264, |
| "kl": 0.00024546185159124434, |
| "learning_rate": 1.142857142857143e-06, |
| "loss": -0.0013, |
| "num_tokens": 58915.0, |
| "reward": 0.30358171463012695, |
| "reward_std": 0.5374526381492615, |
| "rewards/rollout_reward_func/mean": 0.30358171463012695, |
| "rewards/rollout_reward_func/std": 0.7121029496192932, |
| "sampling/importance_sampling_ratio/max": 0.09015928953886032, |
| "sampling/importance_sampling_ratio/mean": 0.028123818337917328, |
| "sampling/importance_sampling_ratio/min": 6.7218247539707566e-18, |
| "sampling/sampling_logp_difference/max": 3.8256945610046387, |
| "sampling/sampling_logp_difference/mean": 0.9157212376594543, |
| "step": 5, |
| "step_time": 5.45849032499973 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.3711864948272705, |
| "epoch": 6e-05, |
| "grad_norm": 0.014959807507693768, |
| "kl": 0.00025912884848366957, |
| "learning_rate": 1.4285714285714286e-06, |
| "loss": -0.0013, |
| "step": 6, |
| "step_time": 2.0148870949979028 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 290.0, |
| "completions/max_terminated_length": 290.0, |
| "completions/mean_length": 159.78125, |
| "completions/mean_terminated_length": 159.78125, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 6.184065341949463, |
| "epoch": 7e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.020354198291897774, |
| "kl": 0.00016448212227260228, |
| "learning_rate": 1.7142857142857145e-06, |
| "loss": 0.0001, |
| "num_tokens": 77924.0, |
| "reward": 0.4891502559185028, |
| "reward_std": 0.6122604012489319, |
| "rewards/rollout_reward_func/mean": 0.4891502559185028, |
| "rewards/rollout_reward_func/std": 0.7439028024673462, |
| "sampling/importance_sampling_ratio/max": 0.23023700714111328, |
| "sampling/importance_sampling_ratio/mean": 0.04043475538492203, |
| "sampling/importance_sampling_ratio/min": 0.0005593057139776647, |
| "sampling/sampling_logp_difference/max": 1.5701570510864258, |
| "sampling/sampling_logp_difference/mean": 0.7449289560317993, |
| "step": 7, |
| "step_time": 4.50355222700091 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.187336444854736, |
| "epoch": 8e-05, |
| "grad_norm": 0.02016047202050686, |
| "kl": 0.00018859418923966587, |
| "learning_rate": 2.0000000000000003e-06, |
| "loss": 0.0001, |
| "step": 8, |
| "step_time": 2.0521075350043247 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 305.0, |
| "completions/max_terminated_length": 305.0, |
| "completions/mean_length": 179.71875, |
| "completions/mean_terminated_length": 179.71875, |
| "completions/min_length": 87.0, |
| "completions/min_terminated_length": 87.0, |
| "entropy": 6.317316174507141, |
| "epoch": 9e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.011887814849615097, |
| "kl": 0.0002640308539412217, |
| "learning_rate": 2.285714285714286e-06, |
| "loss": 0.0001, |
| "num_tokens": 97755.0, |
| "reward": 0.14386901259422302, |
| "reward_std": 0.46610209345817566, |
| "rewards/rollout_reward_func/mean": 0.14386901259422302, |
| "rewards/rollout_reward_func/std": 0.6731327772140503, |
| "sampling/importance_sampling_ratio/max": 0.07798832654953003, |
| "sampling/importance_sampling_ratio/mean": 0.027132228016853333, |
| "sampling/importance_sampling_ratio/min": 3.0175977556363997e-16, |
| "sampling/sampling_logp_difference/max": 3.646486282348633, |
| "sampling/sampling_logp_difference/mean": 0.8435187339782715, |
| "step": 9, |
| "step_time": 4.767648326000199 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.318172335624695, |
| "epoch": 0.0001, |
| "grad_norm": 0.012068596668541431, |
| "kl": 0.00021811862825416028, |
| "learning_rate": 2.571428571428571e-06, |
| "loss": 0.0001, |
| "step": 10, |
| "step_time": 3.0171621790013887 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 305.0, |
| "completions/max_terminated_length": 305.0, |
| "completions/mean_length": 161.15625, |
| "completions/mean_terminated_length": 161.15625, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 6.06863796710968, |
| "epoch": 0.00011, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03053852915763855, |
| "kl": 0.0002923815409303643, |
| "learning_rate": 2.8571428571428573e-06, |
| "loss": 0.0009, |
| "num_tokens": 115960.0, |
| "reward": 0.10083174705505371, |
| "reward_std": 0.73536616563797, |
| "rewards/rollout_reward_func/mean": 0.10083174705505371, |
| "rewards/rollout_reward_func/std": 0.8196083307266235, |
| "sampling/importance_sampling_ratio/max": 0.08715340495109558, |
| "sampling/importance_sampling_ratio/mean": 0.039209067821502686, |
| "sampling/importance_sampling_ratio/min": 0.0009637149632908404, |
| "sampling/sampling_logp_difference/max": 1.487579345703125, |
| "sampling/sampling_logp_difference/mean": 0.7100945711135864, |
| "step": 11, |
| "step_time": 4.602402659003928 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.072819113731384, |
| "epoch": 0.00012, |
| "grad_norm": 0.030046585947275162, |
| "kl": 0.00036826524046773557, |
| "learning_rate": 3.142857142857143e-06, |
| "loss": 0.0009, |
| "step": 12, |
| "step_time": 2.010943456001769 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 348.0, |
| "completions/max_terminated_length": 348.0, |
| "completions/mean_length": 176.90625, |
| "completions/mean_terminated_length": 176.90625, |
| "completions/min_length": 87.0, |
| "completions/min_terminated_length": 87.0, |
| "entropy": 6.142941534519196, |
| "epoch": 0.00013, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012423109263181686, |
| "kl": 0.00031069645046954975, |
| "learning_rate": 3.428571428571429e-06, |
| "loss": 0.0, |
| "num_tokens": 135893.0, |
| "reward": 0.3551899194717407, |
| "reward_std": 0.4100135564804077, |
| "rewards/rollout_reward_func/mean": 0.3551899194717407, |
| "rewards/rollout_reward_func/std": 0.4807114005088806, |
| "sampling/importance_sampling_ratio/max": 0.08160404115915298, |
| "sampling/importance_sampling_ratio/mean": 0.028775818645954132, |
| "sampling/importance_sampling_ratio/min": 9.679094364400953e-05, |
| "sampling/sampling_logp_difference/max": 1.6882461309432983, |
| "sampling/sampling_logp_difference/mean": 0.7273038625717163, |
| "step": 13, |
| "step_time": 4.72503407199838 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.149784088134766, |
| "epoch": 0.00014, |
| "grad_norm": 0.012431415729224682, |
| "kl": 0.00031931480134517187, |
| "learning_rate": 3.7142857142857146e-06, |
| "loss": 0.0001, |
| "step": 14, |
| "step_time": 2.0601124099976005 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 402.0, |
| "completions/max_terminated_length": 402.0, |
| "completions/mean_length": 185.1875, |
| "completions/mean_terminated_length": 187.77418518066406, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 6.218902826309204, |
| "epoch": 0.00015, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.02713136561214924, |
| "kl": 0.00027004735966329463, |
| "learning_rate": 4.000000000000001e-06, |
| "loss": 0.0015, |
| "num_tokens": 155683.0, |
| "reward": 0.305591344833374, |
| "reward_std": 0.40960314869880676, |
| "rewards/rollout_reward_func/mean": 0.305591344833374, |
| "rewards/rollout_reward_func/std": 0.5242363810539246, |
| "sampling/importance_sampling_ratio/max": 0.08599410951137543, |
| "sampling/importance_sampling_ratio/mean": 0.025865234434604645, |
| "sampling/importance_sampling_ratio/min": 7.349282722104652e-14, |
| "sampling/sampling_logp_difference/max": 2.5293257236480713, |
| "sampling/sampling_logp_difference/mean": 0.8104561567306519, |
| "step": 15, |
| "step_time": 4.914467659998991 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.2238587737083435, |
| "epoch": 0.00016, |
| "grad_norm": 0.026739850640296936, |
| "kl": 0.00038530101119249593, |
| "learning_rate": 4.2857142857142855e-06, |
| "loss": 0.0015, |
| "step": 16, |
| "step_time": 2.6130522349994862 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 346.0, |
| "completions/max_terminated_length": 346.0, |
| "completions/mean_length": 161.375, |
| "completions/mean_terminated_length": 161.375, |
| "completions/min_length": 87.0, |
| "completions/min_terminated_length": 87.0, |
| "entropy": 6.213452756404877, |
| "epoch": 0.00017, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03361396864056587, |
| "kl": 0.0007407644588965923, |
| "learning_rate": 4.571428571428572e-06, |
| "loss": 0.0022, |
| "num_tokens": 175471.0, |
| "reward": 0.23945416510105133, |
| "reward_std": 0.7443495988845825, |
| "rewards/rollout_reward_func/mean": 0.23945416510105133, |
| "rewards/rollout_reward_func/std": 0.8175519108772278, |
| "sampling/importance_sampling_ratio/max": 0.0804901197552681, |
| "sampling/importance_sampling_ratio/mean": 0.032769910991191864, |
| "sampling/importance_sampling_ratio/min": 0.0004995061899535358, |
| "sampling/sampling_logp_difference/max": 1.4847469329833984, |
| "sampling/sampling_logp_difference/mean": 0.7405259609222412, |
| "step": 17, |
| "step_time": 4.827751421997164 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.230209648609161, |
| "epoch": 0.00018, |
| "grad_norm": 0.03374752774834633, |
| "kl": 0.001192268646263983, |
| "learning_rate": 4.857142857142858e-06, |
| "loss": 0.0021, |
| "step": 18, |
| "step_time": 2.0522313620022032 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 485.0, |
| "completions/max_terminated_length": 485.0, |
| "completions/mean_length": 174.4375, |
| "completions/mean_terminated_length": 174.4375, |
| "completions/min_length": 87.0, |
| "completions/min_terminated_length": 87.0, |
| "entropy": 6.370483934879303, |
| "epoch": 0.00019, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.018522951751947403, |
| "kl": 0.0011664420198940206, |
| "learning_rate": 5.142857142857142e-06, |
| "loss": -0.0016, |
| "num_tokens": 195133.0, |
| "reward": 0.25696271657943726, |
| "reward_std": 0.5223400592803955, |
| "rewards/rollout_reward_func/mean": 0.25696271657943726, |
| "rewards/rollout_reward_func/std": 0.6738208532333374, |
| "sampling/importance_sampling_ratio/max": 0.07579963654279709, |
| "sampling/importance_sampling_ratio/mean": 0.0267631858587265, |
| "sampling/importance_sampling_ratio/min": 3.3313353689413245e-22, |
| "sampling/sampling_logp_difference/max": 4.017205715179443, |
| "sampling/sampling_logp_difference/mean": 0.8683640956878662, |
| "step": 19, |
| "step_time": 5.342033982999055 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.373503923416138, |
| "epoch": 0.0002, |
| "grad_norm": 0.018172789365053177, |
| "kl": 0.0018378060049144551, |
| "learning_rate": 5.428571428571429e-06, |
| "loss": -0.0016, |
| "step": 20, |
| "step_time": 2.2543999580011587 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 260.0, |
| "completions/max_terminated_length": 260.0, |
| "completions/mean_length": 141.34375, |
| "completions/mean_terminated_length": 141.34375, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 6.3722938895225525, |
| "epoch": 0.00021, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.022796686738729477, |
| "kl": 0.003324791803606786, |
| "learning_rate": 5.7142857142857145e-06, |
| "loss": -0.0005, |
| "num_tokens": 213848.0, |
| "reward": 0.23120194673538208, |
| "reward_std": 0.46816450357437134, |
| "rewards/rollout_reward_func/mean": 0.23120194673538208, |
| "rewards/rollout_reward_func/std": 0.5511780977249146, |
| "sampling/importance_sampling_ratio/max": 0.08035512268543243, |
| "sampling/importance_sampling_ratio/mean": 0.03772790729999542, |
| "sampling/importance_sampling_ratio/min": 9.345225998731152e-18, |
| "sampling/sampling_logp_difference/max": 3.5961461067199707, |
| "sampling/sampling_logp_difference/mean": 0.8969376683235168, |
| "step": 21, |
| "step_time": 4.52679192800133 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.381504118442535, |
| "epoch": 0.00022, |
| "grad_norm": 0.02249361015856266, |
| "kl": 0.00530128157697618, |
| "learning_rate": 6e-06, |
| "loss": -0.0006, |
| "step": 22, |
| "step_time": 2.9375970069977484 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 297.0, |
| "completions/max_terminated_length": 297.0, |
| "completions/mean_length": 131.78125, |
| "completions/mean_terminated_length": 129.06451416015625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 6.41948789358139, |
| "epoch": 0.00023, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02175409533083439, |
| "kl": 0.008126991509925574, |
| "learning_rate": 6.285714285714286e-06, |
| "loss": 0.0015, |
| "num_tokens": 232513.0, |
| "reward": 0.27927568554878235, |
| "reward_std": 0.5316575765609741, |
| "rewards/rollout_reward_func/mean": 0.27927568554878235, |
| "rewards/rollout_reward_func/std": 0.6082769632339478, |
| "sampling/importance_sampling_ratio/max": 0.1689588725566864, |
| "sampling/importance_sampling_ratio/mean": 0.04865079000592232, |
| "sampling/importance_sampling_ratio/min": 8.09948256636731e-16, |
| "sampling/sampling_logp_difference/max": 3.860705852508545, |
| "sampling/sampling_logp_difference/mean": 0.8586986064910889, |
| "step": 23, |
| "step_time": 4.331727726001191 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.4265682101249695, |
| "epoch": 0.00024, |
| "grad_norm": 0.020918752998113632, |
| "kl": 0.011424720054492354, |
| "learning_rate": 6.571428571428572e-06, |
| "loss": 0.0014, |
| "step": 24, |
| "step_time": 2.0076011149994883 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 206.0, |
| "completions/max_terminated_length": 206.0, |
| "completions/mean_length": 107.78125, |
| "completions/mean_terminated_length": 107.78125, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 6.199184834957123, |
| "epoch": 0.00025, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.023662026971578598, |
| "kl": 0.011069391970522702, |
| "learning_rate": 6.857142857142858e-06, |
| "loss": -0.0014, |
| "num_tokens": 250218.0, |
| "reward": 0.21139422059059143, |
| "reward_std": 0.4235047399997711, |
| "rewards/rollout_reward_func/mean": 0.21139422059059143, |
| "rewards/rollout_reward_func/std": 0.5656381249427795, |
| "sampling/importance_sampling_ratio/max": 0.14888890087604523, |
| "sampling/importance_sampling_ratio/mean": 0.06519368290901184, |
| "sampling/importance_sampling_ratio/min": 0.007011369336396456, |
| "sampling/sampling_logp_difference/max": 1.5711028575897217, |
| "sampling/sampling_logp_difference/mean": 0.7155030965805054, |
| "step": 25, |
| "step_time": 4.00336327399782 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.216024577617645, |
| "epoch": 0.00026, |
| "grad_norm": 0.0240799430757761, |
| "kl": 0.01640008925460279, |
| "learning_rate": 7.1428571428571436e-06, |
| "loss": -0.0015, |
| "step": 26, |
| "step_time": 1.988449569002114 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 342.0, |
| "completions/max_terminated_length": 342.0, |
| "completions/mean_length": 181.625, |
| "completions/mean_terminated_length": 181.625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 6.214400053024292, |
| "epoch": 0.00027, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.024456897750496864, |
| "kl": 0.030537182930856943, |
| "learning_rate": 7.428571428571429e-06, |
| "loss": -0.0006, |
| "num_tokens": 269790.0, |
| "reward": 0.027159079909324646, |
| "reward_std": 0.5774486064910889, |
| "rewards/rollout_reward_func/mean": 0.027159079909324646, |
| "rewards/rollout_reward_func/std": 0.5946774482727051, |
| "sampling/importance_sampling_ratio/max": 0.12529020011425018, |
| "sampling/importance_sampling_ratio/mean": 0.028741449117660522, |
| "sampling/importance_sampling_ratio/min": 0.00032201220165006816, |
| "sampling/sampling_logp_difference/max": 1.747678279876709, |
| "sampling/sampling_logp_difference/mean": 0.7429477572441101, |
| "step": 27, |
| "step_time": 4.680500283000583 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.226389408111572, |
| "epoch": 0.00028, |
| "grad_norm": 0.02442527562379837, |
| "kl": 0.045675538247451186, |
| "learning_rate": 7.714285714285716e-06, |
| "loss": -0.0008, |
| "step": 28, |
| "step_time": 2.470751229000598 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 282.0, |
| "completions/max_terminated_length": 282.0, |
| "completions/mean_length": 162.40625, |
| "completions/mean_terminated_length": 162.40625, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 6.057620167732239, |
| "epoch": 0.00029, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.016966896131634712, |
| "kl": 0.04296685662120581, |
| "learning_rate": 8.000000000000001e-06, |
| "loss": -0.0001, |
| "num_tokens": 288771.0, |
| "reward": 0.36989423632621765, |
| "reward_std": 0.14534157514572144, |
| "rewards/rollout_reward_func/mean": 0.36989423632621765, |
| "rewards/rollout_reward_func/std": 0.27579429745674133, |
| "sampling/importance_sampling_ratio/max": 0.121132493019104, |
| "sampling/importance_sampling_ratio/mean": 0.03923218697309494, |
| "sampling/importance_sampling_ratio/min": 0.0010416348231956363, |
| "sampling/sampling_logp_difference/max": 1.7733206748962402, |
| "sampling/sampling_logp_difference/mean": 0.7076070308685303, |
| "step": 29, |
| "step_time": 4.446439086999817 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.042197942733765, |
| "epoch": 0.0003, |
| "grad_norm": 0.017070813104510307, |
| "kl": 0.057768960017710924, |
| "learning_rate": 8.285714285714287e-06, |
| "loss": -0.0, |
| "step": 30, |
| "step_time": 2.0213948890032043 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 338.0, |
| "completions/max_terminated_length": 338.0, |
| "completions/mean_length": 170.75, |
| "completions/mean_terminated_length": 170.75, |
| "completions/min_length": 93.0, |
| "completions/min_terminated_length": 93.0, |
| "entropy": 6.304847180843353, |
| "epoch": 0.00031, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02980108931660652, |
| "kl": 0.0634244168177247, |
| "learning_rate": 8.571428571428571e-06, |
| "loss": -0.003, |
| "num_tokens": 307723.0, |
| "reward": 0.39828288555145264, |
| "reward_std": 0.20638835430145264, |
| "rewards/rollout_reward_func/mean": 0.39828288555145264, |
| "rewards/rollout_reward_func/std": 0.3757359981536865, |
| "sampling/importance_sampling_ratio/max": 0.1449124813079834, |
| "sampling/importance_sampling_ratio/mean": 0.04636143520474434, |
| "sampling/importance_sampling_ratio/min": 1.888638362217978e-16, |
| "sampling/sampling_logp_difference/max": 3.443108558654785, |
| "sampling/sampling_logp_difference/mean": 0.8668484091758728, |
| "step": 31, |
| "step_time": 4.895437238999875 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.228823721408844, |
| "epoch": 0.00032, |
| "grad_norm": 0.031759388744831085, |
| "kl": 0.08063395507633686, |
| "learning_rate": 8.857142857142858e-06, |
| "loss": -0.0031, |
| "step": 32, |
| "step_time": 2.0456357880011637 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 274.0, |
| "completions/max_terminated_length": 274.0, |
| "completions/mean_length": 154.9375, |
| "completions/mean_terminated_length": 154.9375, |
| "completions/min_length": 100.0, |
| "completions/min_terminated_length": 100.0, |
| "entropy": 5.898605585098267, |
| "epoch": 0.00033, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02342640608549118, |
| "kl": 0.09808124136179686, |
| "learning_rate": 9.142857142857144e-06, |
| "loss": -0.0015, |
| "num_tokens": 327073.0, |
| "reward": 0.4524267315864563, |
| "reward_std": 0.3188154101371765, |
| "rewards/rollout_reward_func/mean": 0.4524267315864563, |
| "rewards/rollout_reward_func/std": 0.42413368821144104, |
| "sampling/importance_sampling_ratio/max": 0.1602727472782135, |
| "sampling/importance_sampling_ratio/mean": 0.06595402956008911, |
| "sampling/importance_sampling_ratio/min": 1.780031055886866e-07, |
| "sampling/sampling_logp_difference/max": 2.5631158351898193, |
| "sampling/sampling_logp_difference/mean": 0.6789650917053223, |
| "step": 33, |
| "step_time": 5.394653027000459 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 5.718324422836304, |
| "epoch": 0.00034, |
| "grad_norm": 0.024896224960684776, |
| "kl": 0.11854369938373566, |
| "learning_rate": 9.42857142857143e-06, |
| "loss": -0.0016, |
| "step": 34, |
| "step_time": 2.0065684620003594 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 298.0, |
| "completions/max_terminated_length": 298.0, |
| "completions/mean_length": 188.15625, |
| "completions/mean_terminated_length": 188.15625, |
| "completions/min_length": 87.0, |
| "completions/min_terminated_length": 87.0, |
| "entropy": 5.479340612888336, |
| "epoch": 0.00035, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05328498035669327, |
| "kl": 0.17625079490244389, |
| "learning_rate": 9.714285714285715e-06, |
| "loss": 0.0055, |
| "num_tokens": 347582.0, |
| "reward": 0.6540457010269165, |
| "reward_std": 0.3883731961250305, |
| "rewards/rollout_reward_func/mean": 0.6540457010269165, |
| "rewards/rollout_reward_func/std": 0.5172057151794434, |
| "sampling/importance_sampling_ratio/max": 0.28821438550949097, |
| "sampling/importance_sampling_ratio/mean": 0.05695977061986923, |
| "sampling/importance_sampling_ratio/min": 2.9049160646388295e-11, |
| "sampling/sampling_logp_difference/max": 18.75899314880371, |
| "sampling/sampling_logp_difference/mean": 0.7121621966362, |
| "step": 35, |
| "step_time": 4.564382323002064 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 5.390263617038727, |
| "epoch": 0.00036, |
| "grad_norm": 0.053973328322172165, |
| "kl": 0.1723556686192751, |
| "learning_rate": 1e-05, |
| "loss": 0.0055, |
| "step": 36, |
| "step_time": 2.0383502150034474 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 305.0, |
| "completions/max_terminated_length": 305.0, |
| "completions/mean_length": 144.96875, |
| "completions/mean_terminated_length": 144.96875, |
| "completions/min_length": 93.0, |
| "completions/min_terminated_length": 93.0, |
| "entropy": 5.320340633392334, |
| "epoch": 0.00037, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03199679031968117, |
| "kl": 0.12412955984473228, |
| "learning_rate": 9.9999999995372e-06, |
| "loss": 0.0032, |
| "num_tokens": 366173.0, |
| "reward": 0.6232043504714966, |
| "reward_std": 0.2902255356311798, |
| "rewards/rollout_reward_func/mean": 0.6232043504714966, |
| "rewards/rollout_reward_func/std": 0.4259689748287201, |
| "sampling/importance_sampling_ratio/max": 0.2735931873321533, |
| "sampling/importance_sampling_ratio/mean": 0.11970683187246323, |
| "sampling/importance_sampling_ratio/min": 1.908677859319141e-06, |
| "sampling/sampling_logp_difference/max": 3.270216703414917, |
| "sampling/sampling_logp_difference/mean": 0.5641312599182129, |
| "step": 37, |
| "step_time": 4.327299739001319 |
| }, |
| { |
| "clip_ratio/high_max": 0.010416666977107525, |
| "clip_ratio/high_mean": 0.0052083334885537624, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0052083334885537624, |
| "entropy": 5.401705622673035, |
| "epoch": 0.00038, |
| "grad_norm": 0.030774196609854698, |
| "kl": 0.10921481531113386, |
| "learning_rate": 9.999999998148802e-06, |
| "loss": 0.0031, |
| "step": 38, |
| "step_time": 2.0147021009979653 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 260.0, |
| "completions/max_terminated_length": 260.0, |
| "completions/mean_length": 141.03125, |
| "completions/mean_terminated_length": 141.03125, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 5.776544392108917, |
| "epoch": 0.00039, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.023018015548586845, |
| "kl": 0.10279421554878354, |
| "learning_rate": 9.999999995834804e-06, |
| "loss": -0.0019, |
| "num_tokens": 384990.0, |
| "reward": 0.5750877857208252, |
| "reward_std": 0.20346705615520477, |
| "rewards/rollout_reward_func/mean": 0.5750877857208252, |
| "rewards/rollout_reward_func/std": 0.4033717215061188, |
| "sampling/importance_sampling_ratio/max": 0.2024683803319931, |
| "sampling/importance_sampling_ratio/mean": 0.07642973214387894, |
| "sampling/importance_sampling_ratio/min": 1.974384267189322e-10, |
| "sampling/sampling_logp_difference/max": 3.7110068798065186, |
| "sampling/sampling_logp_difference/mean": 0.6953427791595459, |
| "step": 39, |
| "step_time": 4.782118793002155 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 5.8619155287742615, |
| "epoch": 0.0004, |
| "grad_norm": 0.02198023535311222, |
| "kl": 0.09226133674383163, |
| "learning_rate": 9.999999992595207e-06, |
| "loss": -0.002, |
| "step": 40, |
| "step_time": 2.003476854000837 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 360.0, |
| "completions/max_terminated_length": 360.0, |
| "completions/mean_length": 176.875, |
| "completions/mean_terminated_length": 176.875, |
| "completions/min_length": 93.0, |
| "completions/min_terminated_length": 93.0, |
| "entropy": 5.619787395000458, |
| "epoch": 0.00041, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.02750505320727825, |
| "kl": 0.07494280813261867, |
| "learning_rate": 9.999999988430008e-06, |
| "loss": -0.0014, |
| "num_tokens": 404818.0, |
| "reward": 0.632590115070343, |
| "reward_std": 0.12197521328926086, |
| "rewards/rollout_reward_func/mean": 0.632590115070343, |
| "rewards/rollout_reward_func/std": 0.3571937084197998, |
| "sampling/importance_sampling_ratio/max": 0.18347331881523132, |
| "sampling/importance_sampling_ratio/mean": 0.06567263603210449, |
| "sampling/importance_sampling_ratio/min": 0.0007156870560720563, |
| "sampling/sampling_logp_difference/max": 1.7460063695907593, |
| "sampling/sampling_logp_difference/mean": 0.5835450887680054, |
| "step": 41, |
| "step_time": 4.588236302999576 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 5.655375897884369, |
| "epoch": 0.00042, |
| "grad_norm": 0.028079615905880928, |
| "kl": 0.07384081510826945, |
| "learning_rate": 9.999999983339212e-06, |
| "loss": -0.0015, |
| "step": 42, |
| "step_time": 2.058184131998132 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 338.0, |
| "completions/max_terminated_length": 338.0, |
| "completions/mean_length": 147.3125, |
| "completions/mean_terminated_length": 147.3125, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 5.985348284244537, |
| "epoch": 0.00043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04133318364620209, |
| "kl": 0.07979621039703488, |
| "learning_rate": 9.999999977322818e-06, |
| "loss": -0.0052, |
| "num_tokens": 423108.0, |
| "reward": 0.39953890442848206, |
| "reward_std": 0.2123834490776062, |
| "rewards/rollout_reward_func/mean": 0.39953890442848206, |
| "rewards/rollout_reward_func/std": 0.2874828279018402, |
| "sampling/importance_sampling_ratio/max": 0.1767341047525406, |
| "sampling/importance_sampling_ratio/mean": 0.06585769355297089, |
| "sampling/importance_sampling_ratio/min": 9.253626085648562e-19, |
| "sampling/sampling_logp_difference/max": 3.958195209503174, |
| "sampling/sampling_logp_difference/mean": 0.775503933429718, |
| "step": 43, |
| "step_time": 4.575371188000645 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 5.950318694114685, |
| "epoch": 0.00044, |
| "grad_norm": 0.03985368832945824, |
| "kl": 0.08474360965192318, |
| "learning_rate": 9.999999970380822e-06, |
| "loss": -0.0053, |
| "step": 44, |
| "step_time": 2.516747978001149 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 369.0, |
| "completions/max_terminated_length": 369.0, |
| "completions/mean_length": 193.75, |
| "completions/mean_terminated_length": 193.75, |
| "completions/min_length": 87.0, |
| "completions/min_terminated_length": 87.0, |
| "entropy": 5.851812183856964, |
| "epoch": 0.00045, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03772180527448654, |
| "kl": 0.10888817999511957, |
| "learning_rate": 9.999999962513228e-06, |
| "loss": -0.006, |
| "num_tokens": 442876.0, |
| "reward": 0.6410436630249023, |
| "reward_std": 0.7430479526519775, |
| "rewards/rollout_reward_func/mean": 0.6410436630249023, |
| "rewards/rollout_reward_func/std": 0.8171493411064148, |
| "sampling/importance_sampling_ratio/max": 0.22074148058891296, |
| "sampling/importance_sampling_ratio/mean": 0.05440264195203781, |
| "sampling/importance_sampling_ratio/min": 7.153796750537485e-16, |
| "sampling/sampling_logp_difference/max": 4.365869522094727, |
| "sampling/sampling_logp_difference/mean": 0.780392050743103, |
| "step": 45, |
| "step_time": 5.305774641001335 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 5.684995532035828, |
| "epoch": 0.00046, |
| "grad_norm": 0.039422787725925446, |
| "kl": 0.12296807020902634, |
| "learning_rate": 9.999999953720035e-06, |
| "loss": -0.0063, |
| "step": 46, |
| "step_time": 2.0743733209983475 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 348.0, |
| "completions/max_terminated_length": 348.0, |
| "completions/mean_length": 198.78125, |
| "completions/mean_terminated_length": 198.78125, |
| "completions/min_length": 87.0, |
| "completions/min_terminated_length": 87.0, |
| "entropy": 5.193369805812836, |
| "epoch": 0.00047, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03668779879808426, |
| "kl": 0.12609337456524372, |
| "learning_rate": 9.99999994400124e-06, |
| "loss": -0.0051, |
| "num_tokens": 463485.0, |
| "reward": 0.2012612223625183, |
| "reward_std": 0.6530838012695312, |
| "rewards/rollout_reward_func/mean": 0.2012612223625183, |
| "rewards/rollout_reward_func/std": 0.7242118716239929, |
| "sampling/importance_sampling_ratio/max": 0.31475865840911865, |
| "sampling/importance_sampling_ratio/mean": 0.05936652421951294, |
| "sampling/importance_sampling_ratio/min": 0.00022013229317963123, |
| "sampling/sampling_logp_difference/max": 1.833187460899353, |
| "sampling/sampling_logp_difference/mean": 0.5444918870925903, |
| "step": 47, |
| "step_time": 4.7334515290003765 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 4.908747911453247, |
| "epoch": 0.00048, |
| "grad_norm": 0.03906744718551636, |
| "kl": 0.14741076529026031, |
| "learning_rate": 9.999999933356848e-06, |
| "loss": -0.0055, |
| "step": 48, |
| "step_time": 2.059140684999875 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 290.0, |
| "completions/max_terminated_length": 290.0, |
| "completions/mean_length": 153.5625, |
| "completions/mean_terminated_length": 153.5625, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 4.577030658721924, |
| "epoch": 0.00049, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.03131258487701416, |
| "kl": 0.18377518095076084, |
| "learning_rate": 9.999999921786855e-06, |
| "loss": 0.002, |
| "num_tokens": 482591.0, |
| "reward": 0.41418272256851196, |
| "reward_std": 0.16014696657657623, |
| "rewards/rollout_reward_func/mean": 0.41418272256851196, |
| "rewards/rollout_reward_func/std": 0.3027903735637665, |
| "sampling/importance_sampling_ratio/max": 0.39005929231643677, |
| "sampling/importance_sampling_ratio/mean": 0.15701675415039062, |
| "sampling/importance_sampling_ratio/min": 0.007550857961177826, |
| "sampling/sampling_logp_difference/max": 1.3340117931365967, |
| "sampling/sampling_logp_difference/mean": 0.44514331221580505, |
| "step": 49, |
| "step_time": 4.443075770999712 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 4.31759974360466, |
| "epoch": 0.0005, |
| "grad_norm": 0.03223428502678871, |
| "kl": 0.1999241877347231, |
| "learning_rate": 9.999999909291265e-06, |
| "loss": 0.0021, |
| "step": 50, |
| "step_time": 2.9629076719993463 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 205.0, |
| "completions/max_terminated_length": 205.0, |
| "completions/mean_length": 118.625, |
| "completions/mean_terminated_length": 115.83870697021484, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 4.087187260389328, |
| "epoch": 0.00051, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07074335217475891, |
| "kl": 0.30883508175611496, |
| "learning_rate": 9.999999895870075e-06, |
| "loss": -0.0001, |
| "num_tokens": 500235.0, |
| "reward": 0.6331731081008911, |
| "reward_std": 0.15591080486774445, |
| "rewards/rollout_reward_func/mean": 0.6331731081008911, |
| "rewards/rollout_reward_func/std": 0.26999494433403015, |
| "sampling/importance_sampling_ratio/max": 0.5052566528320312, |
| "sampling/importance_sampling_ratio/mean": 0.21748122572898865, |
| "sampling/importance_sampling_ratio/min": 0.01564314402639866, |
| "sampling/sampling_logp_difference/max": 1.5473060607910156, |
| "sampling/sampling_logp_difference/mean": 0.4317389726638794, |
| "step": 51, |
| "step_time": 4.16781128099683 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 3.906628429889679, |
| "epoch": 0.00052, |
| "grad_norm": 0.07227003574371338, |
| "kl": 0.300492724403739, |
| "learning_rate": 9.999999881523285e-06, |
| "loss": -0.0003, |
| "step": 52, |
| "step_time": 1.9936077560014382 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 286.0, |
| "completions/max_terminated_length": 286.0, |
| "completions/mean_length": 151.84375, |
| "completions/mean_terminated_length": 151.84375, |
| "completions/min_length": 100.0, |
| "completions/min_terminated_length": 100.0, |
| "entropy": 3.3781526386737823, |
| "epoch": 0.00053, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 0.13030877709388733, |
| "kl": 0.2350521795451641, |
| "learning_rate": 9.999999866250896e-06, |
| "loss": 0.012, |
| "num_tokens": 519286.0, |
| "reward": 0.6449999809265137, |
| "reward_std": 0.11053535342216492, |
| "rewards/rollout_reward_func/mean": 0.6449999809265137, |
| "rewards/rollout_reward_func/std": 0.22530117630958557, |
| "sampling/importance_sampling_ratio/max": 0.5501136779785156, |
| "sampling/importance_sampling_ratio/mean": 0.2970014214515686, |
| "sampling/importance_sampling_ratio/min": 0.029202939942479134, |
| "sampling/sampling_logp_difference/max": 0.9902606010437012, |
| "sampling/sampling_logp_difference/mean": 0.2762574255466461, |
| "step": 53, |
| "step_time": 4.454380056002265 |
| }, |
| { |
| "clip_ratio/high_max": 0.028125000186264515, |
| "clip_ratio/high_mean": 0.014062500093132257, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.014062500093132257, |
| "entropy": 3.4167210161685944, |
| "epoch": 0.00054, |
| "grad_norm": 0.11076293885707855, |
| "kl": 0.22031726501882076, |
| "learning_rate": 9.999999850052909e-06, |
| "loss": 0.0108, |
| "step": 54, |
| "step_time": 2.026400898999782 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0625, |
| "completions/max_length": 211.0, |
| "completions/max_terminated_length": 197.0, |
| "completions/mean_length": 155.5, |
| "completions/mean_terminated_length": 155.43333435058594, |
| "completions/min_length": 93.0, |
| "completions/min_terminated_length": 93.0, |
| "entropy": 3.86805123090744, |
| "epoch": 0.00055, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05002529174089432, |
| "kl": 0.17954419739544392, |
| "learning_rate": 9.99999983292932e-06, |
| "loss": -0.0047, |
| "num_tokens": 538070.0, |
| "reward": 0.49569714069366455, |
| "reward_std": 0.242295503616333, |
| "rewards/rollout_reward_func/mean": 0.49569714069366455, |
| "rewards/rollout_reward_func/std": 0.35909703373908997, |
| "sampling/importance_sampling_ratio/max": 0.4999794065952301, |
| "sampling/importance_sampling_ratio/mean": 0.22611427307128906, |
| "sampling/importance_sampling_ratio/min": 2.373902644344905e-17, |
| "sampling/sampling_logp_difference/max": 3.7852988243103027, |
| "sampling/sampling_logp_difference/mean": 0.4584958851337433, |
| "step": 55, |
| "step_time": 4.410495514002832 |
| }, |
| { |
| "clip_ratio/high_max": 0.010416666977107525, |
| "clip_ratio/high_mean": 0.0052083334885537624, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0052083334885537624, |
| "entropy": 3.928067147731781, |
| "epoch": 0.00056, |
| "grad_norm": 0.047599323093891144, |
| "kl": 0.16341876238584518, |
| "learning_rate": 9.999999814880132e-06, |
| "loss": -0.0052, |
| "step": 56, |
| "step_time": 2.9177981660013756 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 342.0, |
| "completions/max_terminated_length": 342.0, |
| "completions/mean_length": 167.96875, |
| "completions/mean_terminated_length": 169.96774291992188, |
| "completions/min_length": 93.0, |
| "completions/min_terminated_length": 93.0, |
| "entropy": 3.8915408849716187, |
| "epoch": 0.00057, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0617266409099102, |
| "kl": 0.16402203310281038, |
| "learning_rate": 9.999999795905347e-06, |
| "loss": -0.0137, |
| "num_tokens": 556741.0, |
| "reward": 0.2770588994026184, |
| "reward_std": 0.5140299201011658, |
| "rewards/rollout_reward_func/mean": 0.2770588994026184, |
| "rewards/rollout_reward_func/std": 0.6300316452980042, |
| "sampling/importance_sampling_ratio/max": 0.4707293212413788, |
| "sampling/importance_sampling_ratio/mean": 0.20163728296756744, |
| "sampling/importance_sampling_ratio/min": 0.009354846552014351, |
| "sampling/sampling_logp_difference/max": 1.20766019821167, |
| "sampling/sampling_logp_difference/mean": 0.3649933934211731, |
| "step": 57, |
| "step_time": 4.600740758000029 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 3.8189143240451813, |
| "epoch": 0.00058, |
| "grad_norm": 0.060559939593076706, |
| "kl": 0.15883306972682476, |
| "learning_rate": 9.999999776004962e-06, |
| "loss": -0.0137, |
| "step": 58, |
| "step_time": 2.0522955199994612 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 297.0, |
| "completions/max_terminated_length": 297.0, |
| "completions/mean_length": 142.46875, |
| "completions/mean_terminated_length": 142.46875, |
| "completions/min_length": 93.0, |
| "completions/min_terminated_length": 93.0, |
| "entropy": 3.5833038985729218, |
| "epoch": 0.00059, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0712868869304657, |
| "kl": 0.1435947585850954, |
| "learning_rate": 9.999999755178978e-06, |
| "loss": -0.0083, |
| "num_tokens": 575084.0, |
| "reward": 0.35289904475212097, |
| "reward_std": 0.3668435215950012, |
| "rewards/rollout_reward_func/mean": 0.35289904475212097, |
| "rewards/rollout_reward_func/std": 0.5060283541679382, |
| "sampling/importance_sampling_ratio/max": 0.449856698513031, |
| "sampling/importance_sampling_ratio/mean": 0.27609291672706604, |
| "sampling/importance_sampling_ratio/min": 0.03381441906094551, |
| "sampling/sampling_logp_difference/max": 0.9775153398513794, |
| "sampling/sampling_logp_difference/mean": 0.3117252588272095, |
| "step": 59, |
| "step_time": 4.590389930997844 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 3.4226994812488556, |
| "epoch": 0.0006, |
| "grad_norm": 0.07168518751859665, |
| "kl": 0.1514890482649207, |
| "learning_rate": 9.999999733427394e-06, |
| "loss": -0.0084, |
| "step": 60, |
| "step_time": 2.024904317002438 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 206.0, |
| "completions/max_terminated_length": 206.0, |
| "completions/mean_length": 145.34375, |
| "completions/mean_terminated_length": 145.34375, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 3.281781107187271, |
| "epoch": 0.00061, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05610927194356918, |
| "kl": 0.18225542083382607, |
| "learning_rate": 9.99999971075021e-06, |
| "loss": 0.0011, |
| "num_tokens": 593311.0, |
| "reward": 0.57869952917099, |
| "reward_std": 0.34909936785697937, |
| "rewards/rollout_reward_func/mean": 0.57869952917099, |
| "rewards/rollout_reward_func/std": 0.538192093372345, |
| "sampling/importance_sampling_ratio/max": 0.585938572883606, |
| "sampling/importance_sampling_ratio/mean": 0.2803860306739807, |
| "sampling/importance_sampling_ratio/min": 0.05912167578935623, |
| "sampling/sampling_logp_difference/max": 0.9743081331253052, |
| "sampling/sampling_logp_difference/mean": 0.2922592759132385, |
| "step": 61, |
| "step_time": 4.679663689001245 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 3.1316112875938416, |
| "epoch": 0.00062, |
| "grad_norm": 0.059742532670497894, |
| "kl": 0.19533881545066833, |
| "learning_rate": 9.999999687147426e-06, |
| "loss": 0.0009, |
| "step": 62, |
| "step_time": 2.4762866889977886 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 258.0, |
| "completions/max_terminated_length": 258.0, |
| "completions/mean_length": 137.0, |
| "completions/mean_terminated_length": 137.0, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 3.003155827522278, |
| "epoch": 0.00063, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11588791012763977, |
| "kl": 0.19905831664800644, |
| "learning_rate": 9.999999662619046e-06, |
| "loss": -0.0209, |
| "num_tokens": 612247.0, |
| "reward": 0.4884374737739563, |
| "reward_std": 0.4819467067718506, |
| "rewards/rollout_reward_func/mean": 0.4884374737739563, |
| "rewards/rollout_reward_func/std": 0.5825604200363159, |
| "sampling/importance_sampling_ratio/max": 0.659873366355896, |
| "sampling/importance_sampling_ratio/mean": 0.3772771954536438, |
| "sampling/importance_sampling_ratio/min": 0.020119559019804, |
| "sampling/sampling_logp_difference/max": 1.0678739547729492, |
| "sampling/sampling_logp_difference/mean": 0.25328272581100464, |
| "step": 63, |
| "step_time": 4.334251456000857 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0078125, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0078125, |
| "entropy": 2.8085409104824066, |
| "epoch": 0.00064, |
| "grad_norm": 0.11190968751907349, |
| "kl": 0.2239690087735653, |
| "learning_rate": 9.999999637165062e-06, |
| "loss": -0.022, |
| "step": 64, |
| "step_time": 2.00917297399792 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 258.0, |
| "completions/max_terminated_length": 258.0, |
| "completions/mean_length": 160.6875, |
| "completions/mean_terminated_length": 160.6875, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 2.5678041726350784, |
| "epoch": 0.00065, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09978003054857254, |
| "kl": 0.235586229711771, |
| "learning_rate": 9.999999610785483e-06, |
| "loss": 0.0039, |
| "num_tokens": 631261.0, |
| "reward": 0.5577644109725952, |
| "reward_std": 0.28208568692207336, |
| "rewards/rollout_reward_func/mean": 0.5577644109725952, |
| "rewards/rollout_reward_func/std": 0.4251752495765686, |
| "sampling/importance_sampling_ratio/max": 0.6934138536453247, |
| "sampling/importance_sampling_ratio/mean": 0.3842087984085083, |
| "sampling/importance_sampling_ratio/min": 0.05158117786049843, |
| "sampling/sampling_logp_difference/max": 1.5275554656982422, |
| "sampling/sampling_logp_difference/mean": 0.22714273631572723, |
| "step": 65, |
| "step_time": 4.462041075998059 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0052083334885537624, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0052083334885537624, |
| "entropy": 2.4012295603752136, |
| "epoch": 0.00066, |
| "grad_norm": 0.10902947187423706, |
| "kl": 0.266902232542634, |
| "learning_rate": 9.999999583480304e-06, |
| "loss": 0.004, |
| "step": 66, |
| "step_time": 1.9998026810044394 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 338.0, |
| "completions/max_terminated_length": 338.0, |
| "completions/mean_length": 183.125, |
| "completions/mean_terminated_length": 183.125, |
| "completions/min_length": 91.0, |
| "completions/min_terminated_length": 91.0, |
| "entropy": 2.563511222600937, |
| "epoch": 0.00067, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09899244457483292, |
| "kl": 0.21912122517824173, |
| "learning_rate": 9.999999555249524e-06, |
| "loss": 0.0031, |
| "num_tokens": 651201.0, |
| "reward": 0.346504807472229, |
| "reward_std": 0.37360942363739014, |
| "rewards/rollout_reward_func/mean": 0.346504807472229, |
| "rewards/rollout_reward_func/std": 0.47065863013267517, |
| "sampling/importance_sampling_ratio/max": 0.5819504857063293, |
| "sampling/importance_sampling_ratio/mean": 0.35113734006881714, |
| "sampling/importance_sampling_ratio/min": 0.023396655917167664, |
| "sampling/sampling_logp_difference/max": 0.822979211807251, |
| "sampling/sampling_logp_difference/mean": 0.21178027987480164, |
| "step": 67, |
| "step_time": 4.976440294996792 |
| }, |
| { |
| "clip_ratio/high_max": 0.008928571827709675, |
| "clip_ratio/high_mean": 0.004464285913854837, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.004464285913854837, |
| "entropy": 2.4892768561840057, |
| "epoch": 0.00068, |
| "grad_norm": 0.0969628244638443, |
| "kl": 0.2153633050620556, |
| "learning_rate": 9.999999526093148e-06, |
| "loss": 0.0029, |
| "step": 68, |
| "step_time": 2.5076440479988378 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 350.0, |
| "completions/max_terminated_length": 350.0, |
| "completions/mean_length": 184.375, |
| "completions/mean_terminated_length": 184.375, |
| "completions/min_length": 91.0, |
| "completions/min_terminated_length": 91.0, |
| "entropy": 2.2106962502002716, |
| "epoch": 0.00069, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.16426467895507812, |
| "kl": 0.3220958895981312, |
| "learning_rate": 9.999999496011169e-06, |
| "loss": -0.0045, |
| "num_tokens": 671269.0, |
| "reward": 0.7414458990097046, |
| "reward_std": 0.2716183662414551, |
| "rewards/rollout_reward_func/mean": 0.7414458990097046, |
| "rewards/rollout_reward_func/std": 0.4371891915798187, |
| "sampling/importance_sampling_ratio/max": 0.7719125747680664, |
| "sampling/importance_sampling_ratio/mean": 0.45400264859199524, |
| "sampling/importance_sampling_ratio/min": 0.031057482585310936, |
| "sampling/sampling_logp_difference/max": 1.5289125442504883, |
| "sampling/sampling_logp_difference/mean": 0.18032191693782806, |
| "step": 69, |
| "step_time": 4.6545613879952725 |
| }, |
| { |
| "clip_ratio/high_max": 0.030357143841683865, |
| "clip_ratio/high_mean": 0.015178571920841932, |
| "clip_ratio/low_mean": 0.004464285913854837, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.01964285783469677, |
| "entropy": 2.1326727122068405, |
| "epoch": 0.0007, |
| "grad_norm": 0.12904977798461914, |
| "kl": 0.3180929869413376, |
| "learning_rate": 9.999999465003593e-06, |
| "loss": -0.0057, |
| "step": 70, |
| "step_time": 2.0704947099984565 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 210.0, |
| "completions/max_terminated_length": 210.0, |
| "completions/mean_length": 175.84375, |
| "completions/mean_terminated_length": 175.84375, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 2.0988260358572006, |
| "epoch": 0.00071, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3079679012298584, |
| "kl": 0.2244853936135769, |
| "learning_rate": 9.999999433070417e-06, |
| "loss": -0.0203, |
| "num_tokens": 691168.0, |
| "reward": 0.5884735584259033, |
| "reward_std": 0.47160544991493225, |
| "rewards/rollout_reward_func/mean": 0.5884735584259033, |
| "rewards/rollout_reward_func/std": 0.6704621911048889, |
| "sampling/importance_sampling_ratio/max": 0.638032853603363, |
| "sampling/importance_sampling_ratio/mean": 0.4377201199531555, |
| "sampling/importance_sampling_ratio/min": 0.1587381660938263, |
| "sampling/sampling_logp_difference/max": 0.907859206199646, |
| "sampling/sampling_logp_difference/mean": 0.15975390374660492, |
| "step": 71, |
| "step_time": 4.186460018994694 |
| }, |
| { |
| "clip_ratio/high_max": 0.07083333469927311, |
| "clip_ratio/high_mean": 0.05208333441987634, |
| "clip_ratio/low_mean": 0.04062500083819032, |
| "clip_ratio/low_min": 0.02083333395421505, |
| "clip_ratio/region_mean": 0.09270833572372794, |
| "entropy": 1.9240455776453018, |
| "epoch": 0.00072, |
| "grad_norm": 0.22433313727378845, |
| "kl": 0.23141289688646793, |
| "learning_rate": 9.999999400211643e-06, |
| "loss": -0.0249, |
| "step": 72, |
| "step_time": 1.9681833920003555 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 262.0, |
| "completions/max_terminated_length": 262.0, |
| "completions/mean_length": 168.46875, |
| "completions/mean_terminated_length": 168.46875, |
| "completions/min_length": 97.0, |
| "completions/min_terminated_length": 97.0, |
| "entropy": 1.7944766581058502, |
| "epoch": 0.00073, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3287738561630249, |
| "kl": 0.27973234094679356, |
| "learning_rate": 9.99999936642727e-06, |
| "loss": -0.0086, |
| "num_tokens": 711023.0, |
| "reward": 0.5628845691680908, |
| "reward_std": 0.23407763242721558, |
| "rewards/rollout_reward_func/mean": 0.5628845691680908, |
| "rewards/rollout_reward_func/std": 0.41712766885757446, |
| "sampling/importance_sampling_ratio/max": 0.8928364515304565, |
| "sampling/importance_sampling_ratio/mean": 0.5339679718017578, |
| "sampling/importance_sampling_ratio/min": 0.031287215650081635, |
| "sampling/sampling_logp_difference/max": 2.0212931632995605, |
| "sampling/sampling_logp_difference/mean": 0.14184574782848358, |
| "step": 73, |
| "step_time": 5.250511515001563 |
| }, |
| { |
| "clip_ratio/high_max": 0.07500000111758709, |
| "clip_ratio/high_mean": 0.054166668094694614, |
| "clip_ratio/low_mean": 0.11197916977107525, |
| "clip_ratio/low_min": 0.056250001303851604, |
| "clip_ratio/region_mean": 0.16614583879709244, |
| "entropy": 1.5351449102163315, |
| "epoch": 0.00074, |
| "grad_norm": 0.1667291224002838, |
| "kl": 0.3220853917300701, |
| "learning_rate": 9.999999331717294e-06, |
| "loss": -0.0118, |
| "step": 74, |
| "step_time": 1.9973168569995323 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 305.0, |
| "completions/max_terminated_length": 305.0, |
| "completions/mean_length": 187.75, |
| "completions/mean_terminated_length": 187.75, |
| "completions/min_length": 93.0, |
| "completions/min_terminated_length": 93.0, |
| "entropy": 1.4904573261737823, |
| "epoch": 0.00075, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.34346434473991394, |
| "kl": 0.2825224921107292, |
| "learning_rate": 9.999999296081722e-06, |
| "loss": 0.035, |
| "num_tokens": 730815.0, |
| "reward": 0.3780096173286438, |
| "reward_std": 0.4338669776916504, |
| "rewards/rollout_reward_func/mean": 0.3780096173286438, |
| "rewards/rollout_reward_func/std": 0.6683818101882935, |
| "sampling/importance_sampling_ratio/max": 1.004560947418213, |
| "sampling/importance_sampling_ratio/mean": 0.5335597991943359, |
| "sampling/importance_sampling_ratio/min": 9.286272306674448e-10, |
| "sampling/sampling_logp_difference/max": 18.32341766357422, |
| "sampling/sampling_logp_difference/mean": 0.260251522064209, |
| "step": 75, |
| "step_time": 4.602757915003167 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 200000, |
| "num_input_tokens_seen": 730815, |
| "num_train_epochs": 2, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
