| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.00053, |
| "eval_steps": 500, |
| "global_step": 53, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 3.0, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 0.48936397582292557, |
| "epoch": 1e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0005664514610543847, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": 0.0, |
| "num_tokens": 146997.0, |
| "reward": -3.158337116241455, |
| "reward_std": 11.52173900604248, |
| "rewards/rollout_reward_func/mean": -3.158337116241455, |
| "rewards/rollout_reward_func/std": 11.52173900604248, |
| "sampling/importance_sampling_ratio/max": 0.9331093430519104, |
| "sampling/importance_sampling_ratio/mean": 0.8982968330383301, |
| "sampling/importance_sampling_ratio/min": 0.8507044315338135, |
| "sampling/sampling_logp_difference/max": 0.15258322656154633, |
| "sampling/sampling_logp_difference/mean": 0.03600664064288139, |
| "step": 1, |
| "step_time": 78.72238844999993 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.48936397582292557, |
| "epoch": 2e-05, |
| "grad_norm": 0.0005677085719071329, |
| "kl": 0.0, |
| "learning_rate": 2.2857142857142855e-07, |
| "loss": 0.0, |
| "step": 2, |
| "step_time": 15.87692344900006 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.010416666977107525, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.96875, |
| "completions/mean_terminated_length": 2.96875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.48732541501522064, |
| "epoch": 3e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.002515916945412755, |
| "kl": 0.003698288830491947, |
| "learning_rate": 4.571428571428571e-07, |
| "loss": -0.0, |
| "num_tokens": 300422.0, |
| "reward": -3.6841847896575928, |
| "reward_std": 11.474555015563965, |
| "rewards/rollout_reward_func/mean": -3.6841847896575928, |
| "rewards/rollout_reward_func/std": 11.474555015563965, |
| "sampling/importance_sampling_ratio/max": 0.9328096508979797, |
| "sampling/importance_sampling_ratio/mean": 0.8980345726013184, |
| "sampling/importance_sampling_ratio/min": 0.8055793642997742, |
| "sampling/sampling_logp_difference/max": 0.14251333475112915, |
| "sampling/sampling_logp_difference/mean": 0.03639774024486542, |
| "step": 3, |
| "step_time": 80.83202714800018 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.47548339515924454, |
| "epoch": 4e-05, |
| "grad_norm": 0.0018481042934581637, |
| "kl": 0.0001756823347314196, |
| "learning_rate": 6.857142857142857e-07, |
| "loss": -0.0, |
| "step": 4, |
| "step_time": 16.098328456000672 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.875, |
| "completions/mean_terminated_length": 2.875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.5311543047428131, |
| "epoch": 5e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0028014357667416334, |
| "kl": 0.0004833272454334292, |
| "learning_rate": 9.142857142857142e-07, |
| "loss": 0.0001, |
| "num_tokens": 456990.0, |
| "reward": -1.6541011333465576, |
| "reward_std": 11.05872631072998, |
| "rewards/rollout_reward_func/mean": -1.6541011333465576, |
| "rewards/rollout_reward_func/std": 11.058727264404297, |
| "sampling/importance_sampling_ratio/max": 0.9697506427764893, |
| "sampling/importance_sampling_ratio/mean": 0.8795021772384644, |
| "sampling/importance_sampling_ratio/min": 0.5391229391098022, |
| "sampling/sampling_logp_difference/max": 0.4758959412574768, |
| "sampling/sampling_logp_difference/mean": 0.04838065803050995, |
| "step": 5, |
| "step_time": 86.28426266100041 |
| }, |
| { |
| "clip_ratio/high_max": 0.02500000037252903, |
| "clip_ratio/high_mean": 0.012500000186264515, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.012500000186264515, |
| "entropy": 0.5359498672187328, |
| "epoch": 6e-05, |
| "grad_norm": 0.0022959199268370867, |
| "kl": 0.002978163921227406, |
| "learning_rate": 1.1428571428571428e-06, |
| "loss": 0.0001, |
| "step": 6, |
| "step_time": 16.73348437599998 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 3.0, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 0.5112962238490582, |
| "epoch": 7e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.00931193120777607, |
| "kl": 5.794192431451961e-05, |
| "learning_rate": 1.3714285714285715e-06, |
| "loss": 0.0, |
| "num_tokens": 606759.0, |
| "reward": -1.4654988050460815, |
| "reward_std": 12.649130821228027, |
| "rewards/rollout_reward_func/mean": -1.4654988050460815, |
| "rewards/rollout_reward_func/std": 12.649128913879395, |
| "sampling/importance_sampling_ratio/max": 1.0540623664855957, |
| "sampling/importance_sampling_ratio/mean": 0.9033976197242737, |
| "sampling/importance_sampling_ratio/min": 0.8677928447723389, |
| "sampling/sampling_logp_difference/max": 0.17156457901000977, |
| "sampling/sampling_logp_difference/mean": 0.037677086889743805, |
| "step": 7, |
| "step_time": 80.9153498870005 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.5051367729902267, |
| "epoch": 8e-05, |
| "grad_norm": 0.004157723858952522, |
| "kl": 4.364301761228262e-05, |
| "learning_rate": 1.6e-06, |
| "loss": 0.0, |
| "step": 8, |
| "step_time": 16.163697053000305 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.875, |
| "completions/mean_terminated_length": 2.875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.5464204885065556, |
| "epoch": 9e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.003582538105547428, |
| "kl": 0.0011944683974434156, |
| "learning_rate": 1.8285714285714284e-06, |
| "loss": 0.0, |
| "num_tokens": 757249.0, |
| "reward": -7.697338104248047, |
| "reward_std": 8.61632251739502, |
| "rewards/rollout_reward_func/mean": -7.697338104248047, |
| "rewards/rollout_reward_func/std": 8.61632251739502, |
| "sampling/importance_sampling_ratio/max": 1.1465768814086914, |
| "sampling/importance_sampling_ratio/mean": 0.9066257476806641, |
| "sampling/importance_sampling_ratio/min": 0.8192122578620911, |
| "sampling/sampling_logp_difference/max": 0.23310279846191406, |
| "sampling/sampling_logp_difference/mean": 0.042134545743465424, |
| "step": 9, |
| "step_time": 87.67375510900001 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.012500000186264515, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.012500000186264515, |
| "entropy": 0.5470796823501587, |
| "epoch": 0.0001, |
| "grad_norm": 0.004420289304107428, |
| "kl": 0.0015031041582567184, |
| "learning_rate": 2.057142857142857e-06, |
| "loss": -0.0, |
| "step": 10, |
| "step_time": 16.138566152999374 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.010416666977107525, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 3.0, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 0.4877597391605377, |
| "epoch": 0.00011, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0015756486682221293, |
| "kl": 0.0009349783420873337, |
| "learning_rate": 2.2857142857142856e-06, |
| "loss": 0.0, |
| "num_tokens": 910271.0, |
| "reward": -1.2807047367095947, |
| "reward_std": 11.581576347351074, |
| "rewards/rollout_reward_func/mean": -1.2807047367095947, |
| "rewards/rollout_reward_func/std": 11.581576347351074, |
| "sampling/importance_sampling_ratio/max": 1.075101375579834, |
| "sampling/importance_sampling_ratio/mean": 0.9127196669578552, |
| "sampling/importance_sampling_ratio/min": 0.7523821592330933, |
| "sampling/sampling_logp_difference/max": 0.17058521509170532, |
| "sampling/sampling_logp_difference/mean": 0.04037006199359894, |
| "step": 11, |
| "step_time": 84.74441413100021 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.48288414254784584, |
| "epoch": 0.00012, |
| "grad_norm": 0.0031028895173221827, |
| "kl": 0.00019705418833382282, |
| "learning_rate": 2.5142857142857142e-06, |
| "loss": 0.0, |
| "step": 12, |
| "step_time": 16.036360359999435 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 4.0, |
| "completions/max_terminated_length": 4.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.49491142481565475, |
| "epoch": 0.00013, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0027884214650839567, |
| "kl": 0.0024777476012687316, |
| "learning_rate": 2.742857142857143e-06, |
| "loss": 0.0, |
| "num_tokens": 1049539.0, |
| "reward": -7.930685997009277, |
| "reward_std": 10.647638320922852, |
| "rewards/rollout_reward_func/mean": -7.930685997009277, |
| "rewards/rollout_reward_func/std": 10.647637367248535, |
| "sampling/importance_sampling_ratio/max": 1.1072638034820557, |
| "sampling/importance_sampling_ratio/mean": 0.9023596048355103, |
| "sampling/importance_sampling_ratio/min": 0.5414906740188599, |
| "sampling/sampling_logp_difference/max": 0.4979434013366699, |
| "sampling/sampling_logp_difference/mean": 0.0404733270406723, |
| "step": 13, |
| "step_time": 80.93550676899986 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.503012377768755, |
| "epoch": 0.00014, |
| "grad_norm": 0.002723120851442218, |
| "kl": 0.00012674138329771267, |
| "learning_rate": 2.9714285714285716e-06, |
| "loss": 0.0, |
| "step": 14, |
| "step_time": 16.415520546000607 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.96875, |
| "completions/mean_terminated_length": 2.96875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.5103708207607269, |
| "epoch": 0.00015, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0012367322342470288, |
| "kl": 0.0009814446365226104, |
| "learning_rate": 3.2e-06, |
| "loss": -0.0, |
| "num_tokens": 1198467.0, |
| "reward": -1.7910524606704712, |
| "reward_std": 14.345645904541016, |
| "rewards/rollout_reward_func/mean": -1.7910524606704712, |
| "rewards/rollout_reward_func/std": 14.345645904541016, |
| "sampling/importance_sampling_ratio/max": 0.9826624393463135, |
| "sampling/importance_sampling_ratio/mean": 0.9046387672424316, |
| "sampling/importance_sampling_ratio/min": 0.8640578985214233, |
| "sampling/sampling_logp_difference/max": 0.1460454910993576, |
| "sampling/sampling_logp_difference/mean": 0.03705396130681038, |
| "step": 15, |
| "step_time": 84.01811012399958 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.5053780749440193, |
| "epoch": 0.00016, |
| "grad_norm": 0.0012754682684317231, |
| "kl": 0.0003874969786465954, |
| "learning_rate": 3.428571428571428e-06, |
| "loss": -0.0, |
| "step": 16, |
| "step_time": 16.14125642999943 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 4.0, |
| "completions/max_terminated_length": 4.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 3.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.4758913367986679, |
| "epoch": 0.00017, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.000835731509141624, |
| "kl": 2.2824605100169038e-05, |
| "learning_rate": 3.657142857142857e-06, |
| "loss": -0.0, |
| "num_tokens": 1352552.0, |
| "reward": -2.025902271270752, |
| "reward_std": 9.88846492767334, |
| "rewards/rollout_reward_func/mean": -2.025902271270752, |
| "rewards/rollout_reward_func/std": 9.88846492767334, |
| "sampling/importance_sampling_ratio/max": 0.9374787211418152, |
| "sampling/importance_sampling_ratio/mean": 0.9018987417221069, |
| "sampling/importance_sampling_ratio/min": 0.8059737086296082, |
| "sampling/sampling_logp_difference/max": 0.1600051373243332, |
| "sampling/sampling_logp_difference/mean": 0.034846384078264236, |
| "step": 17, |
| "step_time": 87.90823437400013 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.47047749534249306, |
| "epoch": 0.00018, |
| "grad_norm": 0.000915601325687021, |
| "kl": 3.488661536721338e-05, |
| "learning_rate": 3.885714285714286e-06, |
| "loss": -0.0, |
| "step": 18, |
| "step_time": 16.648166115000095 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 4.0, |
| "completions/max_terminated_length": 4.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 3.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.4496545232832432, |
| "epoch": 0.00019, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0004858207830693573, |
| "kl": 3.536299266215792e-05, |
| "learning_rate": 4.114285714285714e-06, |
| "loss": 0.0, |
| "num_tokens": 1501094.0, |
| "reward": -3.7144558429718018, |
| "reward_std": 11.072718620300293, |
| "rewards/rollout_reward_func/mean": -3.7144558429718018, |
| "rewards/rollout_reward_func/std": 11.072717666625977, |
| "sampling/importance_sampling_ratio/max": 0.9308063983917236, |
| "sampling/importance_sampling_ratio/mean": 0.899206817150116, |
| "sampling/importance_sampling_ratio/min": 0.66890549659729, |
| "sampling/sampling_logp_difference/max": 0.32890522480010986, |
| "sampling/sampling_logp_difference/mean": 0.03617248684167862, |
| "step": 19, |
| "step_time": 80.7782728230004 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.44183528050780296, |
| "epoch": 0.0002, |
| "grad_norm": 0.00042393436888232827, |
| "kl": 0.00015307544185816369, |
| "learning_rate": 4.342857142857142e-06, |
| "loss": 0.0, |
| "step": 20, |
| "step_time": 16.09888975399963 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.5056380964815617, |
| "epoch": 0.00021, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.00032198833650909364, |
| "kl": 4.869016663633374e-05, |
| "learning_rate": 4.571428571428571e-06, |
| "loss": -0.0, |
| "num_tokens": 1651955.0, |
| "reward": -1.5917177200317383, |
| "reward_std": 13.17538070678711, |
| "rewards/rollout_reward_func/mean": -1.5917177200317383, |
| "rewards/rollout_reward_func/std": 13.175381660461426, |
| "sampling/importance_sampling_ratio/max": 0.9360762238502502, |
| "sampling/importance_sampling_ratio/mean": 0.9005730152130127, |
| "sampling/importance_sampling_ratio/min": 0.5624815821647644, |
| "sampling/sampling_logp_difference/max": 0.6438815593719482, |
| "sampling/sampling_logp_difference/mean": 0.038462258875370026, |
| "step": 21, |
| "step_time": 86.13508222099972 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.4966563805937767, |
| "epoch": 0.00022, |
| "grad_norm": 0.0003532824048306793, |
| "kl": 8.798080261840369e-05, |
| "learning_rate": 4.8e-06, |
| "loss": -0.0, |
| "step": 22, |
| "step_time": 16.589822670999638 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.96875, |
| "completions/mean_terminated_length": 2.96875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.46145135909318924, |
| "epoch": 0.00023, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0017623284365981817, |
| "kl": 8.294408689835109e-05, |
| "learning_rate": 5.0285714285714285e-06, |
| "loss": 0.0, |
| "num_tokens": 1801431.0, |
| "reward": -3.1079812049865723, |
| "reward_std": 15.382396697998047, |
| "rewards/rollout_reward_func/mean": -3.1079812049865723, |
| "rewards/rollout_reward_func/std": 15.382396697998047, |
| "sampling/importance_sampling_ratio/max": 0.9533551335334778, |
| "sampling/importance_sampling_ratio/mean": 0.9110425710678101, |
| "sampling/importance_sampling_ratio/min": 0.8689562678337097, |
| "sampling/sampling_logp_difference/max": 0.12607532739639282, |
| "sampling/sampling_logp_difference/mean": 0.031991466879844666, |
| "step": 23, |
| "step_time": 84.70673029099953 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.4546354450285435, |
| "epoch": 0.00024, |
| "grad_norm": 0.002184377983212471, |
| "kl": 0.00013984292854729574, |
| "learning_rate": 5.257142857142857e-06, |
| "loss": 0.0, |
| "step": 24, |
| "step_time": 15.989325058999839 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.50193877145648, |
| "epoch": 0.00025, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0028643528930842876, |
| "kl": 0.0006277600778048509, |
| "learning_rate": 5.485714285714286e-06, |
| "loss": -0.0, |
| "num_tokens": 1954111.0, |
| "reward": -1.1967151165008545, |
| "reward_std": 9.904693603515625, |
| "rewards/rollout_reward_func/mean": -1.1967151165008545, |
| "rewards/rollout_reward_func/std": 9.904692649841309, |
| "sampling/importance_sampling_ratio/max": 1.0367523431777954, |
| "sampling/importance_sampling_ratio/mean": 0.9070800542831421, |
| "sampling/importance_sampling_ratio/min": 0.641558825969696, |
| "sampling/sampling_logp_difference/max": 0.2708436846733093, |
| "sampling/sampling_logp_difference/mean": 0.03570127859711647, |
| "step": 25, |
| "step_time": 88.23927604999926 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.493778470903635, |
| "epoch": 0.00026, |
| "grad_norm": 0.002861190587282181, |
| "kl": 0.006509271008326323, |
| "learning_rate": 5.7142857142857145e-06, |
| "loss": -0.0, |
| "step": 26, |
| "step_time": 17.19145845399953 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.90625, |
| "completions/mean_terminated_length": 2.90625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.4369519129395485, |
| "epoch": 0.00027, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0023894549813121557, |
| "kl": 0.00021573404501395999, |
| "learning_rate": 5.942857142857143e-06, |
| "loss": 0.0, |
| "num_tokens": 2103766.0, |
| "reward": -4.149303436279297, |
| "reward_std": 11.891597747802734, |
| "rewards/rollout_reward_func/mean": -4.149303436279297, |
| "rewards/rollout_reward_func/std": 11.891597747802734, |
| "sampling/importance_sampling_ratio/max": 0.9366415739059448, |
| "sampling/importance_sampling_ratio/mean": 0.9109276533126831, |
| "sampling/importance_sampling_ratio/min": 0.7386661171913147, |
| "sampling/sampling_logp_difference/max": 0.17405521869659424, |
| "sampling/sampling_logp_difference/mean": 0.03263023495674133, |
| "step": 27, |
| "step_time": 84.90730395599985 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.4287884756922722, |
| "epoch": 0.00028, |
| "grad_norm": 0.003158987034112215, |
| "kl": 0.0010838782172868378, |
| "learning_rate": 6.171428571428571e-06, |
| "loss": 0.0, |
| "step": 28, |
| "step_time": 16.073362338999686 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.90625, |
| "completions/mean_terminated_length": 2.90625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.47590041905641556, |
| "epoch": 0.00029, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0019009847892448306, |
| "kl": 0.002789211330764374, |
| "learning_rate": 6.4e-06, |
| "loss": -0.0, |
| "num_tokens": 2255958.0, |
| "reward": -1.561665415763855, |
| "reward_std": 11.948278427124023, |
| "rewards/rollout_reward_func/mean": -1.561665415763855, |
| "rewards/rollout_reward_func/std": 11.948278427124023, |
| "sampling/importance_sampling_ratio/max": 0.951849639415741, |
| "sampling/importance_sampling_ratio/mean": 0.8972554802894592, |
| "sampling/importance_sampling_ratio/min": 0.328483909368515, |
| "sampling/sampling_logp_difference/max": 1.0063753128051758, |
| "sampling/sampling_logp_difference/mean": 0.04340764880180359, |
| "step": 29, |
| "step_time": 91.08275033600057 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.47105172276496887, |
| "epoch": 0.0003, |
| "grad_norm": 0.002664634957909584, |
| "kl": 0.0012541578576019674, |
| "learning_rate": 6.628571428571428e-06, |
| "loss": -0.0, |
| "step": 30, |
| "step_time": 16.112164762000702 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 3.0, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 0.3540297709405422, |
| "epoch": 0.00031, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.00012217683251947165, |
| "kl": 0.00030874026197125204, |
| "learning_rate": 6.857142857142856e-06, |
| "loss": -0.0, |
| "num_tokens": 2407027.0, |
| "reward": 0.8819369077682495, |
| "reward_std": 9.408540725708008, |
| "rewards/rollout_reward_func/mean": 0.8819369077682495, |
| "rewards/rollout_reward_func/std": 9.408540725708008, |
| "sampling/importance_sampling_ratio/max": 0.9496375322341919, |
| "sampling/importance_sampling_ratio/mean": 0.926586389541626, |
| "sampling/importance_sampling_ratio/min": 0.8927135467529297, |
| "sampling/sampling_logp_difference/max": 0.11497166007757187, |
| "sampling/sampling_logp_difference/mean": 0.025536321103572845, |
| "step": 31, |
| "step_time": 87.01734963399849 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.34124791249632835, |
| "epoch": 0.00032, |
| "grad_norm": 0.00012294144835323095, |
| "kl": 0.0003481047842797125, |
| "learning_rate": 7.085714285714285e-06, |
| "loss": -0.0, |
| "step": 32, |
| "step_time": 16.119427530000394 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.37663908302783966, |
| "epoch": 0.00033, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.00028033516719006, |
| "kl": 0.00011349991063980269, |
| "learning_rate": 7.314285714285714e-06, |
| "loss": 0.0, |
| "num_tokens": 2556831.0, |
| "reward": -5.249267101287842, |
| "reward_std": 9.296294212341309, |
| "rewards/rollout_reward_func/mean": -5.249267101287842, |
| "rewards/rollout_reward_func/std": 9.296294212341309, |
| "sampling/importance_sampling_ratio/max": 0.9602704644203186, |
| "sampling/importance_sampling_ratio/mean": 0.9252135753631592, |
| "sampling/importance_sampling_ratio/min": 0.8558757305145264, |
| "sampling/sampling_logp_difference/max": 0.1554594784975052, |
| "sampling/sampling_logp_difference/mean": 0.027311213314533234, |
| "step": 33, |
| "step_time": 91.05606818099977 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.37676023691892624, |
| "epoch": 0.00034, |
| "grad_norm": 0.0002865525020752102, |
| "kl": 0.00011139449998154305, |
| "learning_rate": 7.542857142857142e-06, |
| "loss": 0.0, |
| "step": 34, |
| "step_time": 16.66090228600069 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.010416666977107525, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 3.0, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 0.38403602689504623, |
| "epoch": 0.00035, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0007655026274733245, |
| "kl": 0.0008573867726227036, |
| "learning_rate": 7.771428571428572e-06, |
| "loss": 0.0, |
| "num_tokens": 2711341.0, |
| "reward": -3.2598910331726074, |
| "reward_std": 11.653153419494629, |
| "rewards/rollout_reward_func/mean": -3.2598910331726074, |
| "rewards/rollout_reward_func/std": 11.653153419494629, |
| "sampling/importance_sampling_ratio/max": 1.3437409400939941, |
| "sampling/importance_sampling_ratio/mean": 0.9472247362136841, |
| "sampling/importance_sampling_ratio/min": 0.872779130935669, |
| "sampling/sampling_logp_difference/max": 0.351446270942688, |
| "sampling/sampling_logp_difference/mean": 0.03257140517234802, |
| "step": 35, |
| "step_time": 97.56342971100003 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.010416666977107525, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "entropy": 0.37373727560043335, |
| "epoch": 0.00036, |
| "grad_norm": 0.0005624265759252012, |
| "kl": 0.0011794225774792721, |
| "learning_rate": 8e-06, |
| "loss": 0.0, |
| "step": 36, |
| "step_time": 16.08771912999964 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 3.0, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 0.3612252026796341, |
| "epoch": 0.00037, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.003778834594413638, |
| "kl": 0.005491923870067694, |
| "learning_rate": 7.999999999958871e-06, |
| "loss": 0.0, |
| "num_tokens": 2860979.0, |
| "reward": -3.204911947250366, |
| "reward_std": 13.321104049682617, |
| "rewards/rollout_reward_func/mean": -3.204911947250366, |
| "rewards/rollout_reward_func/std": 13.3211030960083, |
| "sampling/importance_sampling_ratio/max": 0.9534703493118286, |
| "sampling/importance_sampling_ratio/mean": 0.9272962808609009, |
| "sampling/importance_sampling_ratio/min": 0.812048077583313, |
| "sampling/sampling_logp_difference/max": 0.125631183385849, |
| "sampling/sampling_logp_difference/mean": 0.027381815016269684, |
| "step": 37, |
| "step_time": 93.92803074699941 |
| }, |
| { |
| "clip_ratio/high_max": 0.0416666679084301, |
| "clip_ratio/high_mean": 0.02083333395421505, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.02083333395421505, |
| "entropy": 0.3561956323683262, |
| "epoch": 0.00038, |
| "grad_norm": 0.00041312171379104257, |
| "kl": 0.009066745038580848, |
| "learning_rate": 7.999999999835487e-06, |
| "loss": -0.0, |
| "step": 38, |
| "step_time": 16.57034048200012 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.90625, |
| "completions/mean_terminated_length": 2.90625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.37458185851573944, |
| "epoch": 0.00039, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.003013370558619499, |
| "kl": 0.0015790614006618853, |
| "learning_rate": 7.999999999629846e-06, |
| "loss": -0.0, |
| "num_tokens": 3014870.0, |
| "reward": -4.495431900024414, |
| "reward_std": 9.237488746643066, |
| "rewards/rollout_reward_func/mean": -4.495431900024414, |
| "rewards/rollout_reward_func/std": 9.237488746643066, |
| "sampling/importance_sampling_ratio/max": 0.955286979675293, |
| "sampling/importance_sampling_ratio/mean": 0.9233821630477905, |
| "sampling/importance_sampling_ratio/min": 0.7778754234313965, |
| "sampling/sampling_logp_difference/max": 0.19711723923683167, |
| "sampling/sampling_logp_difference/mean": 0.028256624937057495, |
| "step": 39, |
| "step_time": 95.70763315500017 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.3661472350358963, |
| "epoch": 0.0004, |
| "grad_norm": 0.0022348016500473022, |
| "kl": 0.0027103804022772238, |
| "learning_rate": 7.99999999934195e-06, |
| "loss": -0.0, |
| "step": 40, |
| "step_time": 16.061642451000353 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.96875, |
| "completions/mean_terminated_length": 2.96875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.28179030306637287, |
| "epoch": 0.00041, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 6.525115895783529e-05, |
| "kl": 0.000415502121541067, |
| "learning_rate": 7.999999998971795e-06, |
| "loss": -0.0, |
| "num_tokens": 3164817.0, |
| "reward": 2.3517534732818604, |
| "reward_std": 12.381294250488281, |
| "rewards/rollout_reward_func/mean": 2.3517534732818604, |
| "rewards/rollout_reward_func/std": 12.381294250488281, |
| "sampling/importance_sampling_ratio/max": 0.9573846459388733, |
| "sampling/importance_sampling_ratio/mean": 0.9431747198104858, |
| "sampling/importance_sampling_ratio/min": 0.9145121574401855, |
| "sampling/sampling_logp_difference/max": 0.08903493732213974, |
| "sampling/sampling_logp_difference/mean": 0.01973060891032219, |
| "step": 41, |
| "step_time": 90.856857966 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.27641875483095646, |
| "epoch": 0.00042, |
| "grad_norm": 0.0001371208782074973, |
| "kl": 0.0004361617029644549, |
| "learning_rate": 7.999999998519386e-06, |
| "loss": -0.0, |
| "step": 42, |
| "step_time": 16.129788360000475 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.90625, |
| "completions/mean_terminated_length": 2.90625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.32416083104908466, |
| "epoch": 0.00043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0014907444128766656, |
| "kl": 0.0035448235757939983, |
| "learning_rate": 7.99999999798472e-06, |
| "loss": 0.0, |
| "num_tokens": 3317992.0, |
| "reward": 0.9361296892166138, |
| "reward_std": 10.966468811035156, |
| "rewards/rollout_reward_func/mean": 0.9361296892166138, |
| "rewards/rollout_reward_func/std": 10.966468811035156, |
| "sampling/importance_sampling_ratio/max": 1.039656639099121, |
| "sampling/importance_sampling_ratio/mean": 0.9434545636177063, |
| "sampling/importance_sampling_ratio/min": 0.8878768682479858, |
| "sampling/sampling_logp_difference/max": 0.11892110854387283, |
| "sampling/sampling_logp_difference/mean": 0.022497989237308502, |
| "step": 43, |
| "step_time": 92.78864051999926 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.3210602290928364, |
| "epoch": 0.00044, |
| "grad_norm": 0.0009183208458125591, |
| "kl": 0.0044001892892993055, |
| "learning_rate": 7.999999997367799e-06, |
| "loss": -0.0, |
| "step": 44, |
| "step_time": 16.18974080999942 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.96875, |
| "completions/mean_terminated_length": 2.96875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.2824931964278221, |
| "epoch": 0.00045, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.005496473051607609, |
| "kl": 0.09349646368718822, |
| "learning_rate": 7.999999996668619e-06, |
| "loss": 0.0, |
| "num_tokens": 3472410.0, |
| "reward": 1.872360348701477, |
| "reward_std": 12.642463684082031, |
| "rewards/rollout_reward_func/mean": 1.872360348701477, |
| "rewards/rollout_reward_func/std": 12.642463684082031, |
| "sampling/importance_sampling_ratio/max": 0.9575082063674927, |
| "sampling/importance_sampling_ratio/mean": 0.941848635673523, |
| "sampling/importance_sampling_ratio/min": 0.8221862316131592, |
| "sampling/sampling_logp_difference/max": 0.12410736083984375, |
| "sampling/sampling_logp_difference/mean": 0.0203759353607893, |
| "step": 45, |
| "step_time": 92.56621814599839 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.010416666977107525, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "entropy": 0.2739644553512335, |
| "epoch": 0.00046, |
| "grad_norm": 0.0008409882429987192, |
| "kl": 0.17168781322834548, |
| "learning_rate": 7.999999995887185e-06, |
| "loss": 0.0, |
| "step": 46, |
| "step_time": 16.97712109299937 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.90625, |
| "completions/mean_terminated_length": 2.90625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.3127005323767662, |
| "epoch": 0.00047, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.005147633608430624, |
| "kl": 0.02404813828979968, |
| "learning_rate": 7.999999995023493e-06, |
| "loss": 0.0, |
| "num_tokens": 3623971.0, |
| "reward": -2.005354404449463, |
| "reward_std": 9.463374137878418, |
| "rewards/rollout_reward_func/mean": -2.005354404449463, |
| "rewards/rollout_reward_func/std": 9.463374137878418, |
| "sampling/importance_sampling_ratio/max": 1.8077476024627686, |
| "sampling/importance_sampling_ratio/mean": 0.9680489897727966, |
| "sampling/importance_sampling_ratio/min": 0.8782174587249756, |
| "sampling/sampling_logp_difference/max": 0.6487047672271729, |
| "sampling/sampling_logp_difference/mean": 0.027968257665634155, |
| "step": 47, |
| "step_time": 90.81133413299995 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "entropy": 0.31450361758470535, |
| "epoch": 0.00048, |
| "grad_norm": 0.010047705844044685, |
| "kl": 0.011364400590537116, |
| "learning_rate": 7.999999994077545e-06, |
| "loss": -0.0, |
| "step": 48, |
| "step_time": 16.131004170999404 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.31293279118835926, |
| "epoch": 0.00049, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0025514012668281794, |
| "kl": 0.004930681967380224, |
| "learning_rate": 7.999999993049342e-06, |
| "loss": -0.0, |
| "num_tokens": 3769693.0, |
| "reward": -1.6621102094650269, |
| "reward_std": 11.216950416564941, |
| "rewards/rollout_reward_func/mean": -1.6621102094650269, |
| "rewards/rollout_reward_func/std": 11.216950416564941, |
| "sampling/importance_sampling_ratio/max": 0.963409960269928, |
| "sampling/importance_sampling_ratio/mean": 0.9157481789588928, |
| "sampling/importance_sampling_ratio/min": 0.4775127172470093, |
| "sampling/sampling_logp_difference/max": 0.6763087511062622, |
| "sampling/sampling_logp_difference/mean": 0.03370527923107147, |
| "step": 49, |
| "step_time": 87.11181479400057 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.02083333395421505, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.02083333395421505, |
| "entropy": 0.305842787027359, |
| "epoch": 0.0005, |
| "grad_norm": 0.001049969345331192, |
| "kl": 0.01695527902484173, |
| "learning_rate": 7.999999991938882e-06, |
| "loss": -0.0, |
| "step": 50, |
| "step_time": 17.37328599800003 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.26627207547426224, |
| "epoch": 0.00051, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.000988983316347003, |
| "kl": 0.0010694206866901368, |
| "learning_rate": 7.999999990746166e-06, |
| "loss": -0.0, |
| "num_tokens": 3918993.0, |
| "reward": 0.07662695646286011, |
| "reward_std": 12.704819679260254, |
| "rewards/rollout_reward_func/mean": 0.07662695646286011, |
| "rewards/rollout_reward_func/std": 12.704818725585938, |
| "sampling/importance_sampling_ratio/max": 0.9753377437591553, |
| "sampling/importance_sampling_ratio/mean": 0.950916051864624, |
| "sampling/importance_sampling_ratio/min": 0.8742276430130005, |
| "sampling/sampling_logp_difference/max": 0.08313722163438797, |
| "sampling/sampling_logp_difference/mean": 0.018695753067731857, |
| "step": 51, |
| "step_time": 84.65364402499836 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.26170745119452477, |
| "epoch": 0.00052, |
| "grad_norm": 0.0009646639809943736, |
| "kl": 0.0012894975807284936, |
| "learning_rate": 7.999999989471194e-06, |
| "loss": -0.0, |
| "step": 52, |
| "step_time": 15.9307113390023 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.96875, |
| "completions/mean_terminated_length": 2.96875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.26491483487188816, |
| "epoch": 0.00053, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0002617822028696537, |
| "kl": 0.0023044159715936985, |
| "learning_rate": 7.999999988113964e-06, |
| "loss": -0.0, |
| "num_tokens": 4071645.0, |
| "reward": -0.05424162745475769, |
| "reward_std": 9.241598129272461, |
| "rewards/rollout_reward_func/mean": -0.05424162745475769, |
| "rewards/rollout_reward_func/std": 9.241597175598145, |
| "sampling/importance_sampling_ratio/max": 0.9647132754325867, |
| "sampling/importance_sampling_ratio/mean": 0.9405180215835571, |
| "sampling/importance_sampling_ratio/min": 0.6948662400245667, |
| "sampling/sampling_logp_difference/max": 0.30833685398101807, |
| "sampling/sampling_logp_difference/mean": 0.021141711622476578, |
| "step": 53, |
| "step_time": 93.0931882169998 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 600000, |
| "num_input_tokens_seen": 4071645, |
| "num_train_epochs": 6, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
