| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.0006, |
| "eval_steps": 500, |
| "global_step": 60, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 2.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.0, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 8.54889988899231, |
| "epoch": 1e-05, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.03602602332830429, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": -0.0011, |
| "num_tokens": 37068.0, |
| "reward": -0.20241966843605042, |
| "reward_std": 0.48188072443008423, |
| "rewards/rollout_reward_func/mean": -0.20241966843605042, |
| "rewards/rollout_reward_func/std": 0.5643806457519531, |
| "sampling/importance_sampling_ratio/max": 0.07946816831827164, |
| "sampling/importance_sampling_ratio/mean": 0.04483851045370102, |
| "sampling/importance_sampling_ratio/min": 0.013508557341992855, |
| "sampling/sampling_logp_difference/max": 2.2406487464904785, |
| "sampling/sampling_logp_difference/mean": 1.6288080215454102, |
| "step": 1, |
| "step_time": 5.987766452998585 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 8.54889988899231, |
| "epoch": 2e-05, |
| "grad_norm": 0.03546445816755295, |
| "kl": 0.0, |
| "learning_rate": 2.8571428571428575e-07, |
| "loss": -0.0011, |
| "step": 2, |
| "step_time": 2.7118166719992587 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 2.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.0, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 8.612898111343384, |
| "epoch": 3e-05, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.028961816802620888, |
| "kl": 0.0006349515169858932, |
| "learning_rate": 5.714285714285715e-07, |
| "loss": -0.0011, |
| "num_tokens": 73706.0, |
| "reward": -0.41709062457084656, |
| "reward_std": 0.38032135367393494, |
| "rewards/rollout_reward_func/mean": -0.41709062457084656, |
| "rewards/rollout_reward_func/std": 0.5864750146865845, |
| "sampling/importance_sampling_ratio/max": 0.07319076359272003, |
| "sampling/importance_sampling_ratio/mean": 0.04485338553786278, |
| "sampling/importance_sampling_ratio/min": 0.010412708856165409, |
| "sampling/sampling_logp_difference/max": 2.62746524810791, |
| "sampling/sampling_logp_difference/mean": 1.631974458694458, |
| "step": 3, |
| "step_time": 5.237267270999837 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 8.621538519859314, |
| "epoch": 4e-05, |
| "grad_norm": 0.027420829981565475, |
| "kl": 0.0009405575692653656, |
| "learning_rate": 8.571428571428572e-07, |
| "loss": -0.0011, |
| "step": 4, |
| "step_time": 2.7475997250003275 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 5.0, |
| "completions/mean_length": 2.53125, |
| "completions/mean_terminated_length": 2.096774101257324, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 8.608618140220642, |
| "epoch": 5e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04249607026576996, |
| "kl": 0.0009066914208233356, |
| "learning_rate": 1.142857142857143e-06, |
| "loss": -0.0019, |
| "num_tokens": 108621.0, |
| "reward": -0.4377398192882538, |
| "reward_std": 0.6512277126312256, |
| "rewards/rollout_reward_func/mean": -0.4377398192882538, |
| "rewards/rollout_reward_func/std": 0.8918678760528564, |
| "sampling/importance_sampling_ratio/max": 0.0737227350473404, |
| "sampling/importance_sampling_ratio/mean": 0.03858622536063194, |
| "sampling/importance_sampling_ratio/min": 1.0664673354490728e-12, |
| "sampling/sampling_logp_difference/max": 4.4544358253479, |
| "sampling/sampling_logp_difference/mean": 1.7643263339996338, |
| "step": 5, |
| "step_time": 6.21450720699886 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 8.600322604179382, |
| "epoch": 6e-05, |
| "grad_norm": 0.04186399653553963, |
| "kl": 0.0009581162594258785, |
| "learning_rate": 1.4285714285714286e-06, |
| "loss": -0.002, |
| "step": 6, |
| "step_time": 2.7577716289997625 |
| }, |
| { |
| "clip_ratio/high_max": 0.013888888992369175, |
| "clip_ratio/high_mean": 0.0069444444961845875, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0069444444961845875, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7.0, |
| "completions/max_terminated_length": 7.0, |
| "completions/mean_length": 2.15625, |
| "completions/mean_terminated_length": 2.15625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 8.887233972549438, |
| "epoch": 7e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01845015399158001, |
| "kl": 0.001277084978937637, |
| "learning_rate": 1.7142857142857145e-06, |
| "loss": -0.0001, |
| "num_tokens": 142642.0, |
| "reward": -0.24416625499725342, |
| "reward_std": 0.7532411813735962, |
| "rewards/rollout_reward_func/mean": -0.24416625499725342, |
| "rewards/rollout_reward_func/std": 0.8073007464408875, |
| "sampling/importance_sampling_ratio/max": 0.05460228770971298, |
| "sampling/importance_sampling_ratio/mean": 0.028035324066877365, |
| "sampling/importance_sampling_ratio/min": 1.4402675105884555e-06, |
| "sampling/sampling_logp_difference/max": 3.2391209602355957, |
| "sampling/sampling_logp_difference/mean": 1.8511791229248047, |
| "step": 7, |
| "step_time": 5.192916050999884 |
| }, |
| { |
| "clip_ratio/high_max": 0.02777777798473835, |
| "clip_ratio/high_mean": 0.013888888992369175, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.013888888992369175, |
| "entropy": 8.885694026947021, |
| "epoch": 8e-05, |
| "grad_norm": 0.018649807199835777, |
| "kl": 0.0008755421440582722, |
| "learning_rate": 2.0000000000000003e-06, |
| "loss": -0.0001, |
| "step": 8, |
| "step_time": 2.734651068999483 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.4375, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 8.617319464683533, |
| "epoch": 9e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03367479518055916, |
| "kl": 0.0011472837650217116, |
| "learning_rate": 2.285714285714286e-06, |
| "loss": -0.0017, |
| "num_tokens": 180338.0, |
| "reward": -0.2762250006198883, |
| "reward_std": 0.6308639645576477, |
| "rewards/rollout_reward_func/mean": -0.2762250006198883, |
| "rewards/rollout_reward_func/std": 0.619946300983429, |
| "sampling/importance_sampling_ratio/max": 0.08318282663822174, |
| "sampling/importance_sampling_ratio/mean": 0.04022577404975891, |
| "sampling/importance_sampling_ratio/min": 8.560036707239149e-11, |
| "sampling/sampling_logp_difference/max": 2.8513259887695312, |
| "sampling/sampling_logp_difference/mean": 1.6676621437072754, |
| "step": 9, |
| "step_time": 5.192849637998734 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 8.604341268539429, |
| "epoch": 0.0001, |
| "grad_norm": 0.03420183062553406, |
| "kl": 0.001177078731416259, |
| "learning_rate": 2.571428571428571e-06, |
| "loss": -0.0017, |
| "step": 10, |
| "step_time": 2.779620520998833 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 2.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.0, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 8.559729814529419, |
| "epoch": 0.00011, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.02679450251162052, |
| "kl": 0.0014951191842556, |
| "learning_rate": 2.8571428571428573e-06, |
| "loss": -0.0013, |
| "num_tokens": 215884.0, |
| "reward": 0.05839650332927704, |
| "reward_std": 0.5475454926490784, |
| "rewards/rollout_reward_func/mean": 0.05839650332927704, |
| "rewards/rollout_reward_func/std": 0.7081521153450012, |
| "sampling/importance_sampling_ratio/max": 0.09729397296905518, |
| "sampling/importance_sampling_ratio/mean": 0.04614107310771942, |
| "sampling/importance_sampling_ratio/min": 0.009668753482401371, |
| "sampling/sampling_logp_difference/max": 2.5881471633911133, |
| "sampling/sampling_logp_difference/mean": 1.6770999431610107, |
| "step": 11, |
| "step_time": 6.151045407999845 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 8.533240914344788, |
| "epoch": 0.00012, |
| "grad_norm": 0.027015963569283485, |
| "kl": 0.002467602491378784, |
| "learning_rate": 3.142857142857143e-06, |
| "loss": -0.0013, |
| "step": 12, |
| "step_time": 2.712993424999695 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5.0, |
| "completions/max_terminated_length": 5.0, |
| "completions/mean_length": 2.09375, |
| "completions/mean_terminated_length": 2.09375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 8.65865409374237, |
| "epoch": 0.00013, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03438998758792877, |
| "kl": 0.0025057614548131824, |
| "learning_rate": 3.428571428571429e-06, |
| "loss": -0.0012, |
| "num_tokens": 250983.0, |
| "reward": -0.3477444052696228, |
| "reward_std": 0.8011652231216431, |
| "rewards/rollout_reward_func/mean": -0.3477444052696228, |
| "rewards/rollout_reward_func/std": 0.8506107926368713, |
| "sampling/importance_sampling_ratio/max": 0.08270177990198135, |
| "sampling/importance_sampling_ratio/mean": 0.03867126256227493, |
| "sampling/importance_sampling_ratio/min": 2.151069793399074e-06, |
| "sampling/sampling_logp_difference/max": 4.863964557647705, |
| "sampling/sampling_logp_difference/mean": 1.7552906274795532, |
| "step": 13, |
| "step_time": 5.348588517999815 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 8.637501120567322, |
| "epoch": 0.00014, |
| "grad_norm": 0.03334236517548561, |
| "kl": 0.003458394785411656, |
| "learning_rate": 3.7142857142857146e-06, |
| "loss": -0.0013, |
| "step": 14, |
| "step_time": 2.796233564999966 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8.0, |
| "completions/max_terminated_length": 8.0, |
| "completions/mean_length": 2.1875, |
| "completions/mean_terminated_length": 2.1875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 8.605337858200073, |
| "epoch": 0.00015, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0344076044857502, |
| "kl": 0.007246186607517302, |
| "learning_rate": 4.000000000000001e-06, |
| "loss": -0.0011, |
| "num_tokens": 287257.0, |
| "reward": -0.5175027251243591, |
| "reward_std": 0.7213335633277893, |
| "rewards/rollout_reward_func/mean": -0.5175027251243591, |
| "rewards/rollout_reward_func/std": 0.7736045718193054, |
| "sampling/importance_sampling_ratio/max": 0.07941123098134995, |
| "sampling/importance_sampling_ratio/mean": 0.03353440761566162, |
| "sampling/importance_sampling_ratio/min": 8.447619620710611e-05, |
| "sampling/sampling_logp_difference/max": 2.514495372772217, |
| "sampling/sampling_logp_difference/mean": 1.715531349182129, |
| "step": 15, |
| "step_time": 5.0371513690006395 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 8.572421252727509, |
| "epoch": 0.00016, |
| "grad_norm": 0.0361904576420784, |
| "kl": 0.010158459888771176, |
| "learning_rate": 4.2857142857142855e-06, |
| "loss": -0.0012, |
| "step": 16, |
| "step_time": 3.6766240449996985 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0052083334885537624, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0052083334885537624, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 10.0, |
| "completions/mean_length": 2.6875, |
| "completions/mean_terminated_length": 2.2580645084381104, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 8.298084318637848, |
| "epoch": 0.00017, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03764951229095459, |
| "kl": 0.01280841525294818, |
| "learning_rate": 4.571428571428572e-06, |
| "loss": -0.0025, |
| "num_tokens": 319698.0, |
| "reward": -0.4354173243045807, |
| "reward_std": 0.6479126214981079, |
| "rewards/rollout_reward_func/mean": -0.4354173243045807, |
| "rewards/rollout_reward_func/std": 0.8862241506576538, |
| "sampling/importance_sampling_ratio/max": 0.11163407564163208, |
| "sampling/importance_sampling_ratio/mean": 0.04720360040664673, |
| "sampling/importance_sampling_ratio/min": 5.244973292489741e-12, |
| "sampling/sampling_logp_difference/max": 4.713146686553955, |
| "sampling/sampling_logp_difference/mean": 1.6690685749053955, |
| "step": 17, |
| "step_time": 5.1438336040009744 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0052083334885537624, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0052083334885537624, |
| "entropy": 8.210463047027588, |
| "epoch": 0.00018, |
| "grad_norm": 0.03806741535663605, |
| "kl": 0.01803363612270914, |
| "learning_rate": 4.857142857142858e-06, |
| "loss": -0.0026, |
| "step": 18, |
| "step_time": 2.7552450699986366 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 2.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.0, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 8.558752536773682, |
| "epoch": 0.00019, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04838458448648453, |
| "kl": 0.022023072466254234, |
| "learning_rate": 5.142857142857142e-06, |
| "loss": -0.0019, |
| "num_tokens": 351318.0, |
| "reward": -0.6331132054328918, |
| "reward_std": 0.8535451292991638, |
| "rewards/rollout_reward_func/mean": -0.6331132054328918, |
| "rewards/rollout_reward_func/std": 0.9285341501235962, |
| "sampling/importance_sampling_ratio/max": 0.11876487731933594, |
| "sampling/importance_sampling_ratio/mean": 0.04305056482553482, |
| "sampling/importance_sampling_ratio/min": 0.01192709431052208, |
| "sampling/sampling_logp_difference/max": 2.4490432739257812, |
| "sampling/sampling_logp_difference/mean": 1.6915147304534912, |
| "step": 19, |
| "step_time": 4.906003911999505 |
| }, |
| { |
| "clip_ratio/high_max": 0.03125, |
| "clip_ratio/high_mean": 0.015625, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.015625, |
| "entropy": 8.480961680412292, |
| "epoch": 0.0002, |
| "grad_norm": 0.047831419855356216, |
| "kl": 0.03148091956973076, |
| "learning_rate": 5.428571428571429e-06, |
| "loss": -0.0021, |
| "step": 20, |
| "step_time": 2.542041312000947 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 2.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.0, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 7.914846777915955, |
| "epoch": 0.00021, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0461098738014698, |
| "kl": 0.05386248789727688, |
| "learning_rate": 5.7142857142857145e-06, |
| "loss": -0.004, |
| "num_tokens": 382194.0, |
| "reward": 0.12508951127529144, |
| "reward_std": 1.0255252122879028, |
| "rewards/rollout_reward_func/mean": 0.12508951127529144, |
| "rewards/rollout_reward_func/std": 1.0441889762878418, |
| "sampling/importance_sampling_ratio/max": 0.15263937413692474, |
| "sampling/importance_sampling_ratio/mean": 0.07228268682956696, |
| "sampling/importance_sampling_ratio/min": 0.011906024999916553, |
| "sampling/sampling_logp_difference/max": 2.5268516540527344, |
| "sampling/sampling_logp_difference/mean": 1.4949290752410889, |
| "step": 21, |
| "step_time": 5.119830969998475 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 7.746252477169037, |
| "epoch": 0.00022, |
| "grad_norm": 0.04356187954545021, |
| "kl": 0.06908445432782173, |
| "learning_rate": 6e-06, |
| "loss": -0.0043, |
| "step": 22, |
| "step_time": 3.6642874440012747 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 2.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.0, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 7.505157113075256, |
| "epoch": 0.00023, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027521340176463127, |
| "kl": 0.11650293320417404, |
| "learning_rate": 6.285714285714286e-06, |
| "loss": -0.0037, |
| "num_tokens": 418193.0, |
| "reward": 0.050230324268341064, |
| "reward_std": 0.5550512671470642, |
| "rewards/rollout_reward_func/mean": 0.050230324268341064, |
| "rewards/rollout_reward_func/std": 0.7158081531524658, |
| "sampling/importance_sampling_ratio/max": 0.17737992107868195, |
| "sampling/importance_sampling_ratio/mean": 0.08728897571563721, |
| "sampling/importance_sampling_ratio/min": 0.01220005378127098, |
| "sampling/sampling_logp_difference/max": 2.312082290649414, |
| "sampling/sampling_logp_difference/mean": 1.4727132320404053, |
| "step": 23, |
| "step_time": 5.118901103000098 |
| }, |
| { |
| "clip_ratio/high_max": 0.03125, |
| "clip_ratio/high_mean": 0.015625, |
| "clip_ratio/low_mean": 0.03125, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.046875, |
| "entropy": 7.405115187168121, |
| "epoch": 0.00024, |
| "grad_norm": 0.027487488463521004, |
| "kl": 0.18349953554570675, |
| "learning_rate": 6.571428571428572e-06, |
| "loss": -0.0038, |
| "step": 24, |
| "step_time": 2.720049919002122 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.015625, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.015625, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 14.0, |
| "completions/mean_length": 2.8125, |
| "completions/mean_terminated_length": 2.387096643447876, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 7.812176287174225, |
| "epoch": 0.00025, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04480979964137077, |
| "kl": 0.12202088022604585, |
| "learning_rate": 6.857142857142858e-06, |
| "loss": -0.003, |
| "num_tokens": 453429.0, |
| "reward": -0.5567976832389832, |
| "reward_std": 0.5721786022186279, |
| "rewards/rollout_reward_func/mean": -0.5567976832389832, |
| "rewards/rollout_reward_func/std": 0.6177526116371155, |
| "sampling/importance_sampling_ratio/max": 0.19887018203735352, |
| "sampling/importance_sampling_ratio/mean": 0.06785006076097488, |
| "sampling/importance_sampling_ratio/min": 6.38808743654907e-15, |
| "sampling/sampling_logp_difference/max": 4.993244647979736, |
| "sampling/sampling_logp_difference/mean": 1.6287214756011963, |
| "step": 25, |
| "step_time": 5.136989613000878 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 7.775655627250671, |
| "epoch": 0.00026, |
| "grad_norm": 0.051532234996557236, |
| "kl": 0.13951302412897348, |
| "learning_rate": 7.1428571428571436e-06, |
| "loss": -0.0031, |
| "step": 26, |
| "step_time": 2.7561433119990397 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 15.0, |
| "completions/max_terminated_length": 15.0, |
| "completions/mean_length": 2.40625, |
| "completions/mean_terminated_length": 2.40625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 7.139402210712433, |
| "epoch": 0.00027, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1159820482134819, |
| "kl": 0.21849708445370197, |
| "learning_rate": 7.428571428571429e-06, |
| "loss": -0.0033, |
| "num_tokens": 488130.0, |
| "reward": 0.0451994352042675, |
| "reward_std": 0.5705078840255737, |
| "rewards/rollout_reward_func/mean": 0.0451994352042675, |
| "rewards/rollout_reward_func/std": 0.7002898454666138, |
| "sampling/importance_sampling_ratio/max": 0.21661195158958435, |
| "sampling/importance_sampling_ratio/mean": 0.11537822335958481, |
| "sampling/importance_sampling_ratio/min": 1.9111427718154772e-11, |
| "sampling/sampling_logp_difference/max": 5.494892120361328, |
| "sampling/sampling_logp_difference/mean": 1.4119889736175537, |
| "step": 27, |
| "step_time": 5.695921420998275 |
| }, |
| { |
| "clip_ratio/high_max": 0.03125, |
| "clip_ratio/high_mean": 0.015625, |
| "clip_ratio/low_mean": 0.015625, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.03125, |
| "entropy": 7.091515064239502, |
| "epoch": 0.00028, |
| "grad_norm": 0.05888032913208008, |
| "kl": 0.19798127934336662, |
| "learning_rate": 7.714285714285716e-06, |
| "loss": -0.0036, |
| "step": 28, |
| "step_time": 3.262695406999228 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 12.0, |
| "completions/mean_length": 2.84375, |
| "completions/mean_terminated_length": 2.4193546772003174, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 6.829086482524872, |
| "epoch": 0.00029, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.07569188624620438, |
| "kl": 0.3484500087797642, |
| "learning_rate": 8.000000000000001e-06, |
| "loss": -0.0047, |
| "num_tokens": 523810.0, |
| "reward": 0.25646376609802246, |
| "reward_std": 0.19004222750663757, |
| "rewards/rollout_reward_func/mean": 0.25646376609802246, |
| "rewards/rollout_reward_func/std": 0.5962904691696167, |
| "sampling/importance_sampling_ratio/max": 0.2295530140399933, |
| "sampling/importance_sampling_ratio/mean": 0.12002766132354736, |
| "sampling/importance_sampling_ratio/min": 3.580996610352827e-10, |
| "sampling/sampling_logp_difference/max": 4.408937454223633, |
| "sampling/sampling_logp_difference/mean": 1.4061723947525024, |
| "step": 29, |
| "step_time": 5.195359340000323 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.82170957326889, |
| "epoch": 0.0003, |
| "grad_norm": 0.072876937687397, |
| "kl": 0.3563471883535385, |
| "learning_rate": 8.285714285714287e-06, |
| "loss": -0.0048, |
| "step": 30, |
| "step_time": 2.7664619609995498 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.4375, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 7.176153600215912, |
| "epoch": 0.00031, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.012421791441738605, |
| "kl": 0.19252930209040642, |
| "learning_rate": 8.571428571428571e-06, |
| "loss": -0.0031, |
| "num_tokens": 562344.0, |
| "reward": 0.007906697690486908, |
| "reward_std": 0.25298231840133667, |
| "rewards/rollout_reward_func/mean": 0.007906697690486908, |
| "rewards/rollout_reward_func/std": 0.34386366605758667, |
| "sampling/importance_sampling_ratio/max": 0.2527497410774231, |
| "sampling/importance_sampling_ratio/mean": 0.12306913733482361, |
| "sampling/importance_sampling_ratio/min": 1.3953619618263524e-13, |
| "sampling/sampling_logp_difference/max": 4.638323783874512, |
| "sampling/sampling_logp_difference/mean": 1.436547040939331, |
| "step": 31, |
| "step_time": 5.2235907870026494 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 7.177456617355347, |
| "epoch": 0.00032, |
| "grad_norm": 0.011528298258781433, |
| "kl": 0.19802813418209553, |
| "learning_rate": 8.857142857142858e-06, |
| "loss": -0.0031, |
| "step": 32, |
| "step_time": 2.7817133249991457 |
| }, |
| { |
| "clip_ratio/high_max": 0.013888888992369175, |
| "clip_ratio/high_mean": 0.0069444444961845875, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0069444444961845875, |
| "completions/clipped_ratio": 0.0625, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.875, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 7.5588818192481995, |
| "epoch": 0.00033, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.019391389563679695, |
| "kl": 0.19253090023994446, |
| "learning_rate": 9.142857142857144e-06, |
| "loss": -0.0068, |
| "num_tokens": 596903.0, |
| "reward": -0.23663240671157837, |
| "reward_std": 0.7337036728858948, |
| "rewards/rollout_reward_func/mean": -0.23663240671157837, |
| "rewards/rollout_reward_func/std": 0.7588505148887634, |
| "sampling/importance_sampling_ratio/max": 0.24936437606811523, |
| "sampling/importance_sampling_ratio/mean": 0.09548068046569824, |
| "sampling/importance_sampling_ratio/min": 2.8611614813489616e-11, |
| "sampling/sampling_logp_difference/max": 4.682908058166504, |
| "sampling/sampling_logp_difference/mean": 1.4331482648849487, |
| "step": 33, |
| "step_time": 5.650446067998928 |
| }, |
| { |
| "clip_ratio/high_max": 0.045138888992369175, |
| "clip_ratio/high_mean": 0.022569444496184587, |
| "clip_ratio/low_mean": 0.015625, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.03819444449618459, |
| "entropy": 7.561095893383026, |
| "epoch": 0.00034, |
| "grad_norm": 0.0243048295378685, |
| "kl": 0.2237775418907404, |
| "learning_rate": 9.42857142857143e-06, |
| "loss": -0.0068, |
| "step": 34, |
| "step_time": 3.267381875997671 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 2.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.0, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 7.21353954076767, |
| "epoch": 0.00035, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.03242993354797363, |
| "kl": 0.23187845014035702, |
| "learning_rate": 9.714285714285715e-06, |
| "loss": -0.0051, |
| "num_tokens": 630878.0, |
| "reward": 0.07425153255462646, |
| "reward_std": 0.39872053265571594, |
| "rewards/rollout_reward_func/mean": 0.07425153255462646, |
| "rewards/rollout_reward_func/std": 0.5568375587463379, |
| "sampling/importance_sampling_ratio/max": 0.26548439264297485, |
| "sampling/importance_sampling_ratio/mean": 0.12363378703594208, |
| "sampling/importance_sampling_ratio/min": 0.00769618758931756, |
| "sampling/sampling_logp_difference/max": 3.031831979751587, |
| "sampling/sampling_logp_difference/mean": 1.3642754554748535, |
| "step": 35, |
| "step_time": 5.206078858000183 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 7.159900069236755, |
| "epoch": 0.00036, |
| "grad_norm": 0.0284445621073246, |
| "kl": 0.249108312651515, |
| "learning_rate": 1e-05, |
| "loss": -0.0052, |
| "step": 36, |
| "step_time": 2.761536221001734 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 5.0, |
| "completions/mean_length": 2.53125, |
| "completions/mean_terminated_length": 2.096774101257324, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 6.710843145847321, |
| "epoch": 0.00037, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.012415582314133644, |
| "kl": 0.21114499121904373, |
| "learning_rate": 9.9999999995372e-06, |
| "loss": -0.0052, |
| "num_tokens": 668297.0, |
| "reward": -0.05626409500837326, |
| "reward_std": 0.34207823872566223, |
| "rewards/rollout_reward_func/mean": -0.05626409500837326, |
| "rewards/rollout_reward_func/std": 0.40576615929603577, |
| "sampling/importance_sampling_ratio/max": 0.28296810388565063, |
| "sampling/importance_sampling_ratio/mean": 0.16521048545837402, |
| "sampling/importance_sampling_ratio/min": 7.181252052973312e-15, |
| "sampling/sampling_logp_difference/max": 3.9721055030822754, |
| "sampling/sampling_logp_difference/mean": 1.2990117073059082, |
| "step": 37, |
| "step_time": 5.382229439999719 |
| }, |
| { |
| "clip_ratio/high_max": 0.03125, |
| "clip_ratio/high_mean": 0.015625, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.015625, |
| "entropy": 6.636479735374451, |
| "epoch": 0.00038, |
| "grad_norm": 0.011466137133538723, |
| "kl": 0.22695374488830566, |
| "learning_rate": 9.999999998148802e-06, |
| "loss": -0.0053, |
| "step": 38, |
| "step_time": 2.8060927069982426 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.019097222248092294, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.019097222248092294, |
| "completions/clipped_ratio": 0.09375, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 5.0, |
| "completions/mean_length": 3.40625, |
| "completions/mean_terminated_length": 2.1034481525421143, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 6.852519392967224, |
| "epoch": 0.00039, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.024172235280275345, |
| "kl": 0.25472032092511654, |
| "learning_rate": 9.999999995834804e-06, |
| "loss": -0.009, |
| "num_tokens": 704978.0, |
| "reward": -0.23096521198749542, |
| "reward_std": 0.4871903955936432, |
| "rewards/rollout_reward_func/mean": -0.23096521198749542, |
| "rewards/rollout_reward_func/std": 0.5004134774208069, |
| "sampling/importance_sampling_ratio/max": 0.2927253544330597, |
| "sampling/importance_sampling_ratio/mean": 0.1400977373123169, |
| "sampling/importance_sampling_ratio/min": 3.20386295618591e-13, |
| "sampling/sampling_logp_difference/max": 4.607293128967285, |
| "sampling/sampling_logp_difference/mean": 1.322880744934082, |
| "step": 39, |
| "step_time": 5.8398218920001455 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0034722222480922937, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0034722222480922937, |
| "entropy": 6.7819119691848755, |
| "epoch": 0.0004, |
| "grad_norm": 0.023572752252221107, |
| "kl": 0.2616057936102152, |
| "learning_rate": 9.999999992595207e-06, |
| "loss": -0.0092, |
| "step": 40, |
| "step_time": 2.772617318999437 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.03125, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.03125, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.4375, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 6.629358530044556, |
| "epoch": 0.00041, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.021721255034208298, |
| "kl": 0.3800278306007385, |
| "learning_rate": 9.999999988430008e-06, |
| "loss": -0.0068, |
| "num_tokens": 739943.0, |
| "reward": -0.0039390698075294495, |
| "reward_std": 0.5532426238059998, |
| "rewards/rollout_reward_func/mean": -0.0039390698075294495, |
| "rewards/rollout_reward_func/std": 0.6965489983558655, |
| "sampling/importance_sampling_ratio/max": 0.31811973452568054, |
| "sampling/importance_sampling_ratio/mean": 0.16932719945907593, |
| "sampling/importance_sampling_ratio/min": 2.988759240096783e-11, |
| "sampling/sampling_logp_difference/max": 3.9750123023986816, |
| "sampling/sampling_logp_difference/mean": 1.231735110282898, |
| "step": 41, |
| "step_time": 5.137522204998277 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.514784038066864, |
| "epoch": 0.00042, |
| "grad_norm": 0.016385503113269806, |
| "kl": 0.3676511310040951, |
| "learning_rate": 9.999999983339212e-06, |
| "loss": -0.0069, |
| "step": 42, |
| "step_time": 2.7499034580005173 |
| }, |
| { |
| "clip_ratio/high_max": 0.03125, |
| "clip_ratio/high_mean": 0.015625, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.015625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 2.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.0, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 6.131894528865814, |
| "epoch": 0.00043, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.021295806393027306, |
| "kl": 0.3743965122848749, |
| "learning_rate": 9.999999977322818e-06, |
| "loss": -0.0082, |
| "num_tokens": 775585.0, |
| "reward": -0.1512046605348587, |
| "reward_std": 0.5463583469390869, |
| "rewards/rollout_reward_func/mean": -0.1512046605348587, |
| "rewards/rollout_reward_func/std": 0.6555477976799011, |
| "sampling/importance_sampling_ratio/max": 0.33739665150642395, |
| "sampling/importance_sampling_ratio/mean": 0.2010621726512909, |
| "sampling/importance_sampling_ratio/min": 0.013526393100619316, |
| "sampling/sampling_logp_difference/max": 2.7704052925109863, |
| "sampling/sampling_logp_difference/mean": 1.0442215204238892, |
| "step": 43, |
| "step_time": 4.971326774000772 |
| }, |
| { |
| "clip_ratio/high_max": 0.03125, |
| "clip_ratio/high_mean": 0.015625, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.015625, |
| "entropy": 6.021390020847321, |
| "epoch": 0.00044, |
| "grad_norm": 0.024328157305717468, |
| "kl": 0.3970394376665354, |
| "learning_rate": 9.999999970380822e-06, |
| "loss": -0.0084, |
| "step": 44, |
| "step_time": 2.748106813000959 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.4375, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 5.561417460441589, |
| "epoch": 0.00045, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.0709402859210968, |
| "kl": 0.5104918628931046, |
| "learning_rate": 9.999999962513228e-06, |
| "loss": -0.0063, |
| "num_tokens": 813291.0, |
| "reward": 0.015873141586780548, |
| "reward_std": 0.19914725422859192, |
| "rewards/rollout_reward_func/mean": 0.015873141586780548, |
| "rewards/rollout_reward_func/std": 0.27820026874542236, |
| "sampling/importance_sampling_ratio/max": 0.35048311948776245, |
| "sampling/importance_sampling_ratio/mean": 0.24818521738052368, |
| "sampling/importance_sampling_ratio/min": 1.603953254936119e-11, |
| "sampling/sampling_logp_difference/max": 2.631711959838867, |
| "sampling/sampling_logp_difference/mean": 0.938794732093811, |
| "step": 45, |
| "step_time": 5.729797389999476 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 5.534043729305267, |
| "epoch": 0.00046, |
| "grad_norm": 0.07074378430843353, |
| "kl": 0.5150652844458818, |
| "learning_rate": 9.999999953720035e-06, |
| "loss": -0.0064, |
| "step": 46, |
| "step_time": 2.7925665359998675 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0625, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.875, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 6.189878046512604, |
| "epoch": 0.00047, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.018252786248922348, |
| "kl": 0.3964756764471531, |
| "learning_rate": 9.99999994400124e-06, |
| "loss": -0.0065, |
| "num_tokens": 850551.0, |
| "reward": -0.0004236232489347458, |
| "reward_std": 0.22690175473690033, |
| "rewards/rollout_reward_func/mean": -0.0004236232489347458, |
| "rewards/rollout_reward_func/std": 0.31507667899131775, |
| "sampling/importance_sampling_ratio/max": 0.36628416180610657, |
| "sampling/importance_sampling_ratio/mean": 0.19846494495868683, |
| "sampling/importance_sampling_ratio/min": 1.2659386039448606e-10, |
| "sampling/sampling_logp_difference/max": 4.142038822174072, |
| "sampling/sampling_logp_difference/mean": 1.0495840311050415, |
| "step": 47, |
| "step_time": 5.31281194299936 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.272528171539307, |
| "epoch": 0.00048, |
| "grad_norm": 0.01557189505547285, |
| "kl": 0.37549079209566116, |
| "learning_rate": 9.999999933356848e-06, |
| "loss": -0.0065, |
| "step": 48, |
| "step_time": 2.7983943749995888 |
| }, |
| { |
| "clip_ratio/high_max": 0.03125, |
| "clip_ratio/high_mean": 0.015625, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.015625, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.4375, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 5.748881280422211, |
| "epoch": 0.00049, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02466733008623123, |
| "kl": 0.4844143260270357, |
| "learning_rate": 9.999999921786855e-06, |
| "loss": -0.0128, |
| "num_tokens": 888151.0, |
| "reward": -0.03596695885062218, |
| "reward_std": 0.2966480255126953, |
| "rewards/rollout_reward_func/mean": -0.03596695885062218, |
| "rewards/rollout_reward_func/std": 0.41368940472602844, |
| "sampling/importance_sampling_ratio/max": 0.3809582591056824, |
| "sampling/importance_sampling_ratio/mean": 0.24179524183273315, |
| "sampling/importance_sampling_ratio/min": 1.7608540181512922e-09, |
| "sampling/sampling_logp_difference/max": 3.226844310760498, |
| "sampling/sampling_logp_difference/mean": 1.0408389568328857, |
| "step": 49, |
| "step_time": 5.312362946001485 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.03125, |
| "clip_ratio/low_min": 0.03125, |
| "clip_ratio/region_mean": 0.03125, |
| "entropy": 5.748369365930557, |
| "epoch": 0.0005, |
| "grad_norm": 0.022601323202252388, |
| "kl": 0.5012617446482182, |
| "learning_rate": 9.999999909291265e-06, |
| "loss": -0.013, |
| "step": 50, |
| "step_time": 3.2023114370003896 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 2.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.0, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 5.002339065074921, |
| "epoch": 0.00051, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.017942914739251137, |
| "kl": 0.3239247240126133, |
| "learning_rate": 9.999999895870075e-06, |
| "loss": -0.0091, |
| "num_tokens": 926183.0, |
| "reward": 0.09451328217983246, |
| "reward_std": 0.026711096987128258, |
| "rewards/rollout_reward_func/mean": 0.09451328217983246, |
| "rewards/rollout_reward_func/std": 0.038156598806381226, |
| "sampling/importance_sampling_ratio/max": 0.39242714643478394, |
| "sampling/importance_sampling_ratio/mean": 0.2970275282859802, |
| "sampling/importance_sampling_ratio/min": 0.010224375873804092, |
| "sampling/sampling_logp_difference/max": 3.037482500076294, |
| "sampling/sampling_logp_difference/mean": 0.8098665475845337, |
| "step": 51, |
| "step_time": 5.696433805002016 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 5.001863986253738, |
| "epoch": 0.00052, |
| "grad_norm": 0.016663307324051857, |
| "kl": 0.3272556010633707, |
| "learning_rate": 9.999999881523285e-06, |
| "loss": -0.0093, |
| "step": 52, |
| "step_time": 2.7697423050003636 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 2.0, |
| "completions/max_terminated_length": 2.0, |
| "completions/mean_length": 2.0, |
| "completions/mean_terminated_length": 2.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 6.3582839369773865, |
| "epoch": 0.00053, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04750742390751839, |
| "kl": 0.5478413961827755, |
| "learning_rate": 9.999999866250896e-06, |
| "loss": -0.0016, |
| "num_tokens": 960803.0, |
| "reward": -0.5398619174957275, |
| "reward_std": 0.47637394070625305, |
| "rewards/rollout_reward_func/mean": -0.5398619174957275, |
| "rewards/rollout_reward_func/std": 0.5747888684272766, |
| "sampling/importance_sampling_ratio/max": 0.4040625989437103, |
| "sampling/importance_sampling_ratio/mean": 0.21569415926933289, |
| "sampling/importance_sampling_ratio/min": 0.005049743689596653, |
| "sampling/sampling_logp_difference/max": 3.7417404651641846, |
| "sampling/sampling_logp_difference/mean": 1.302799940109253, |
| "step": 53, |
| "step_time": 5.149267007999697 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.336145222187042, |
| "epoch": 0.00054, |
| "grad_norm": 0.038806021213531494, |
| "kl": 0.5272092521190643, |
| "learning_rate": 9.999999850052909e-06, |
| "loss": -0.0016, |
| "step": 54, |
| "step_time": 2.742318255999635 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 4.0, |
| "completions/mean_length": 2.5, |
| "completions/mean_terminated_length": 2.064516067504883, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 5.6335746347904205, |
| "epoch": 0.00055, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029065297916531563, |
| "kl": 0.5286761000752449, |
| "learning_rate": 9.99999983292932e-06, |
| "loss": -0.0135, |
| "num_tokens": 998217.0, |
| "reward": -0.1721905618906021, |
| "reward_std": 0.5463275909423828, |
| "rewards/rollout_reward_func/mean": -0.1721905618906021, |
| "rewards/rollout_reward_func/std": 0.5355534553527832, |
| "sampling/importance_sampling_ratio/max": 0.41675877571105957, |
| "sampling/importance_sampling_ratio/mean": 0.2475878894329071, |
| "sampling/importance_sampling_ratio/min": 2.238332399429055e-07, |
| "sampling/sampling_logp_difference/max": 3.269535541534424, |
| "sampling/sampling_logp_difference/mean": 1.0780588388442993, |
| "step": 55, |
| "step_time": 5.403120343999035 |
| }, |
| { |
| "clip_ratio/high_max": 0.03125, |
| "clip_ratio/high_mean": 0.015625, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.015625, |
| "entropy": 5.5938030779361725, |
| "epoch": 0.00056, |
| "grad_norm": 0.02255568467080593, |
| "kl": 0.4606229439377785, |
| "learning_rate": 9.999999814880132e-06, |
| "loss": -0.0136, |
| "step": 56, |
| "step_time": 3.2759828810003455 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0625, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 6.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 2.133333444595337, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 5.78191477060318, |
| "epoch": 0.00057, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.01589336432516575, |
| "kl": 0.42849645018577576, |
| "learning_rate": 9.999999795905347e-06, |
| "loss": -0.009, |
| "num_tokens": 1031529.0, |
| "reward": -0.07622712850570679, |
| "reward_std": 0.6089849472045898, |
| "rewards/rollout_reward_func/mean": -0.07622712850570679, |
| "rewards/rollout_reward_func/std": 1.0020984411239624, |
| "sampling/importance_sampling_ratio/max": 0.425922691822052, |
| "sampling/importance_sampling_ratio/mean": 0.229965940117836, |
| "sampling/importance_sampling_ratio/min": 6.920892747785956e-09, |
| "sampling/sampling_logp_difference/max": 3.239529609680176, |
| "sampling/sampling_logp_difference/mean": 1.154854416847229, |
| "step": 57, |
| "step_time": 5.6268242130017825 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 5.738904058933258, |
| "epoch": 0.00058, |
| "grad_norm": 0.013298124074935913, |
| "kl": 0.3968820311129093, |
| "learning_rate": 9.999999776004962e-06, |
| "loss": -0.009, |
| "step": 58, |
| "step_time": 2.7349948540004334 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0625, |
| "completions/max_length": 16.0, |
| "completions/max_terminated_length": 13.0, |
| "completions/mean_length": 3.21875, |
| "completions/mean_terminated_length": 2.366666793823242, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 6.8580867648124695, |
| "epoch": 0.00059, |
| "frac_reward_zero_std": 0.25, |
| "grad_norm": 0.013808239251375198, |
| "kl": 0.29788459837436676, |
| "learning_rate": 9.999999755178978e-06, |
| "loss": -0.0089, |
| "num_tokens": 1065697.0, |
| "reward": -0.1516963541507721, |
| "reward_std": 0.6373498439788818, |
| "rewards/rollout_reward_func/mean": -0.1516963541507721, |
| "rewards/rollout_reward_func/std": 0.7994406819343567, |
| "sampling/importance_sampling_ratio/max": 0.4323439598083496, |
| "sampling/importance_sampling_ratio/mean": 0.16014467179775238, |
| "sampling/importance_sampling_ratio/min": 3.285566610444768e-11, |
| "sampling/sampling_logp_difference/max": 3.304318428039551, |
| "sampling/sampling_logp_difference/mean": 1.2896037101745605, |
| "step": 59, |
| "step_time": 5.172425778999241 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 6.81160169839859, |
| "epoch": 0.0006, |
| "grad_norm": 0.012560456059873104, |
| "kl": 0.2846912257373333, |
| "learning_rate": 9.999999733427394e-06, |
| "loss": -0.0089, |
| "step": 60, |
| "step_time": 2.6920222089984236 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 200000, |
| "num_input_tokens_seen": 1065697, |
| "num_train_epochs": 2, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
