{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 500, "global_step": 1758, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.9750000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 829.0, "completions/mean_length": 1019.125, "completions/mean_terminated_length": 829.0, "completions/min_length": 829.0, "completions/min_terminated_length": 829.0, "entropy": 0.15596781112253666, "epoch": 0.0005688282138794084, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.007077466696500778, "kl": 0.0, "learning_rate": 9e-06, "loss": 0.0359, "num_tokens": 113589.0, "reward": 0.02500000037252903, "reward_std": 0.15811388194561005, "rewards/format_reward/mean": 0.02500000037252903, "rewards/format_reward/std": 0.15811389684677124, "rewards/gpt_judge_reward/mean": 0.0, "rewards/gpt_judge_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.579245090484619, "sampling/importance_sampling_ratio/mean": 0.5165095329284668, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.437894344329834, "sampling/sampling_logp_difference/mean": 0.01218413095921278, "step": 1, "step_time": 77.18892928003334 }, { "clip_ratio/high_max": 0.0034863412147387862, "clip_ratio/high_mean": 0.0007495148674934171, "clip_ratio/low_mean": 0.001269531265279511, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0020190461073070765, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.0, "completions/mean_length": 923.7999877929688, "completions/mean_terminated_length": 737.7142944335938, "completions/min_length": 480.0, "completions/min_terminated_length": 480.0, "entropy": 0.08150393515825272, "epoch": 0.0011376564277588168, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.017003362998366356, "kl": 0.0007594254784635268, "learning_rate": 8.994880546075087e-06, "loss": 0.0585, "num_tokens": 223301.0, "reward": 0.675000011920929, "reward_std": 0.9442809820175171, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.393050193786621, "sampling/importance_sampling_ratio/mean": 0.658579409122467, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3470654487609863, "sampling/sampling_logp_difference/mean": 0.008705071173608303, "step": 2, "step_time": 73.9254334399011 }, { "clip_ratio/high_max": 0.0037078340828884393, "clip_ratio/high_mean": 0.0011229190713493153, "clip_ratio/low_mean": 0.0015380859440483619, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026610049681039527, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 813.0, "completions/mean_length": 962.5499877929688, "completions/mean_terminated_length": 532.4000244140625, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.10958387609571218, "epoch": 0.0017064846416382253, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.010546275414526463, "kl": 0.0011413823158363812, "learning_rate": 8.989761092150171e-06, "loss": 0.1231, "num_tokens": 336867.0, "reward": 0.25, "reward_std": 0.6698640584945679, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.6286518573760986, "sampling/importance_sampling_ratio/mean": 0.6015604734420776, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4164440631866455, "sampling/sampling_logp_difference/mean": 0.00984180998057127, "step": 3, "step_time": 76.46910041826777 }, { "clip_ratio/high_max": 0.0069263154291547835, "clip_ratio/high_mean": 0.0017382163787260652, "clip_ratio/low_mean": 0.0017333984578726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034716149093583226, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 941.0, "completions/mean_length": 977.25, "completions/mean_terminated_length": 712.3333740234375, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.13987702410668135, "epoch": 0.0022753128555176336, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02386409416794777, "kl": 0.0009823488362599164, "learning_rate": 8.984641638225256e-06, "loss": -0.0487, "num_tokens": 446389.0, "reward": 0.2750000059604645, "reward_std": 0.6788942813873291, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.9699888229370117, "sampling/importance_sampling_ratio/mean": 0.46834293007850647, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9011240005493164, "sampling/sampling_logp_difference/mean": 0.011997446417808533, "step": 4, "step_time": 73.92625860171393 }, { "clip_ratio/high_max": 0.008091724070254713, "clip_ratio/high_mean": 0.002024465356953442, "clip_ratio/low_mean": 0.0012207031268189894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032451684528496116, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 990.2250366210938, "completions/mean_terminated_length": 873.888916015625, "completions/min_length": 544.0, "completions/min_terminated_length": 544.0, "entropy": 0.15018011629581451, "epoch": 0.002844141069397042, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.012219551019370556, "kl": 0.0009278411234845407, "learning_rate": 8.979522184300342e-06, "loss": 0.0408, "num_tokens": 547470.0, "reward": 0.42500001192092896, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.295233726501465, "sampling/importance_sampling_ratio/mean": 0.5223978757858276, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7859697341918945, "sampling/sampling_logp_difference/mean": 0.011515151709318161, "step": 5, "step_time": 74.72049871901982 }, { "clip_ratio/high_max": 0.010777310526464134, "clip_ratio/high_mean": 0.003559538301487919, "clip_ratio/low_mean": 0.001757812497089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00531735087861307, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 898.0499877929688, "completions/mean_terminated_length": 727.6470336914062, "completions/min_length": 540.0, "completions/min_terminated_length": 540.0, "entropy": 0.09756367467343807, "epoch": 0.0034129692832764505, "frac_reward_zero_std": 0.0, "grad_norm": 0.024188866838812828, "kl": 0.0009187414470943622, "learning_rate": 8.974402730375427e-06, "loss": 0.1382, "num_tokens": 659392.0, "reward": 0.824999988079071, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.1887319087982178, "sampling/importance_sampling_ratio/mean": 0.43954533338546753, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.9894576072692871, "sampling/sampling_logp_difference/mean": 0.009544676169753075, "step": 6, "step_time": 74.66720217443071 }, { "clip_ratio/high_max": 0.005837513541337103, "clip_ratio/high_mean": 0.001579613297508331, "clip_ratio/low_mean": 0.002026367175858468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036059805133845657, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 1005.3500366210938, "completions/mean_terminated_length": 930.75, "completions/min_length": 660.0, "completions/min_terminated_length": 660.0, "entropy": 0.16680051758885384, "epoch": 0.003981797497155859, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.006775145418941975, "kl": 0.0009676802656031214, "learning_rate": 8.969283276450511e-06, "loss": 0.037, "num_tokens": 772662.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.2195560932159424, "sampling/importance_sampling_ratio/mean": 0.4353443682193756, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3382428884506226, "sampling/sampling_logp_difference/mean": 0.012451703660190105, "step": 7, "step_time": 76.51014058291912 }, { "clip_ratio/high_max": 0.003949213132727891, "clip_ratio/high_mean": 0.0007898426410974935, "clip_ratio/low_mean": 0.001123046911743586, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001912889492814429, "completions/clipped_ratio": 0.925000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 671.0, "completions/mean_length": 991.375, "completions/mean_terminated_length": 589.0, "completions/min_length": 521.0, "completions/min_terminated_length": 521.0, "entropy": 0.15198746137320995, "epoch": 0.004550625711035267, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.007470419630408287, "kl": 0.0010765228144009598, "learning_rate": 8.964163822525597e-06, "loss": 0.0373, "num_tokens": 886509.0, "reward": 0.15000000596046448, "reward_std": 0.5334935784339905, "rewards/format_reward/mean": 0.07500000298023224, "rewards/format_reward/std": 0.26674678921699524, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.364121198654175, "sampling/importance_sampling_ratio/mean": 0.39474233984947205, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3736889362335205, "sampling/sampling_logp_difference/mean": 0.012166979722678661, "step": 8, "step_time": 74.94184638326988 }, { "clip_ratio/high_max": 0.003414283797610551, "clip_ratio/high_mean": 0.001150465257524047, "clip_ratio/low_mean": 0.0022216796714928932, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033721449581207708, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 961.0, "completions/mean_length": 996.2750244140625, "completions/mean_terminated_length": 802.2000122070312, "completions/min_length": 561.0, "completions/min_terminated_length": 561.0, "entropy": 0.2169812098145485, "epoch": 0.005119453924914676, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03333505988121033, "kl": 0.0010058161642518826, "learning_rate": 8.959044368600684e-06, "loss": -0.0759, "num_tokens": 999144.0, "reward": 0.25, "reward_std": 0.6698641180992126, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.7500040531158447, "sampling/importance_sampling_ratio/mean": 0.5587627291679382, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3342829942703247, "sampling/sampling_logp_difference/mean": 0.01444447785615921, "step": 9, "step_time": 76.09512388054281 }, { "clip_ratio/high_max": 0.0009791122283786535, "clip_ratio/high_mean": 0.0001958224456757307, "clip_ratio/low_mean": 0.0005859375032741809, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007817599489499116, "completions/clipped_ratio": 0.9750000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 1017.5499877929688, "completions/mean_terminated_length": 766.0, "completions/min_length": 766.0, "completions/min_terminated_length": 766.0, "entropy": 0.09988979250192642, "epoch": 0.005688282138794084, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.004528926685452461, "kl": 0.0009096798385144211, "learning_rate": 8.953924914675768e-06, "loss": 0.0499, "num_tokens": 1113670.0, "reward": 0.05000000074505806, "reward_std": 0.3162277638912201, "rewards/format_reward/mean": 0.02500000037252903, "rewards/format_reward/std": 0.15811389684677124, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.200953245162964, "sampling/importance_sampling_ratio/mean": 0.49759501218795776, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1241332292556763, "sampling/sampling_logp_difference/mean": 0.009314794093370438, "step": 10, "step_time": 73.66690364177339 }, { "clip_ratio/high_max": 0.0017230575904250145, "clip_ratio/high_mean": 0.0003446115297265351, "clip_ratio/low_mean": 0.0011962890712311491, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015409006009576842, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 990.6500244140625, "completions/mean_terminated_length": 875.7777709960938, "completions/min_length": 735.0, "completions/min_terminated_length": 735.0, "entropy": 0.11843304987996817, "epoch": 0.006257110352673493, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.004750149790197611, "kl": 0.0009304960985900834, "learning_rate": 8.948805460750853e-06, "loss": 0.0203, "num_tokens": 1231504.0, "reward": 0.42500001192092896, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.982912302017212, "sampling/importance_sampling_ratio/mean": 0.43258628249168396, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3471739292144775, "sampling/sampling_logp_difference/mean": 0.010787054896354675, "step": 11, "step_time": 78.17629779782146 }, { "clip_ratio/high_max": 0.009353596367873251, "clip_ratio/high_mean": 0.0018707192793954164, "clip_ratio/low_mean": 0.0002598752616904676, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002130594541085884, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 619.0, "completions/mean_length": 914.5250244140625, "completions/mean_terminated_length": 476.625, "completions/min_length": 401.0, "completions/min_terminated_length": 401.0, "entropy": 0.12093847431242466, "epoch": 0.006825938566552901, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.006748368497937918, "kl": 0.0009657698392402381, "learning_rate": 8.94368600682594e-06, "loss": -0.0107, "num_tokens": 1349677.0, "reward": 0.375, "reward_std": 0.7741827964782715, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509575605392456, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.7105265855789185, "sampling/importance_sampling_ratio/mean": 0.3736175000667572, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1211020946502686, "sampling/sampling_logp_difference/mean": 0.01128347497433424, "step": 12, "step_time": 77.74883581208996 }, { "clip_ratio/high_max": 0.00144341797567904, "clip_ratio/high_mean": 0.0002886836009565741, "clip_ratio/low_mean": 0.0009521484389551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012408320399117656, "completions/clipped_ratio": 0.9750000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 866.0, "completions/mean_length": 1020.0499877929688, "completions/mean_terminated_length": 866.0, "completions/min_length": 866.0, "completions/min_terminated_length": 866.0, "entropy": 0.1570674367249012, "epoch": 0.00739476678043231, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.0032405871897935867, "kl": 0.001042454423441086, "learning_rate": 8.938566552901024e-06, "loss": 0.0166, "num_tokens": 1468335.0, "reward": 0.02500000037252903, "reward_std": 0.15811388194561005, "rewards/format_reward/mean": 0.02500000037252903, "rewards/format_reward/std": 0.15811389684677124, "rewards/gpt_judge_reward/mean": 0.0, "rewards/gpt_judge_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3605375289916992, "sampling/importance_sampling_ratio/mean": 0.3145972192287445, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6462743282318115, "sampling/sampling_logp_difference/mean": 0.012904703617095947, "step": 13, "step_time": 76.48353077657521 }, { "clip_ratio/high_max": 0.01007776241749525, "clip_ratio/high_mean": 0.00201555248349905, "clip_ratio/low_mean": 9.765625145519152e-05, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021132087349542417, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 851.0, "completions/mean_length": 953.2999877929688, "completions/mean_terminated_length": 670.5, "completions/min_length": 505.0, "completions/min_terminated_length": 505.0, "entropy": 0.14673491474241018, "epoch": 0.007963594994311717, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.010774620808660984, "kl": 0.0010666156013030559, "learning_rate": 8.93344709897611e-06, "loss": 0.0249, "num_tokens": 1581555.0, "reward": 0.3499999940395355, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.0602867603302, "sampling/importance_sampling_ratio/mean": 0.4038839340209961, "sampling/importance_sampling_ratio/min": 0.021102609112858772, "sampling/sampling_logp_difference/max": 1.4361402988433838, "sampling/sampling_logp_difference/mean": 0.01213639136403799, "step": 14, "step_time": 76.63237355928868 }, { "clip_ratio/high_max": 0.002247191034257412, "clip_ratio/high_mean": 0.0004494382010307163, "clip_ratio/low_mean": 0.0005126953201397555, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009621335211704718, "completions/clipped_ratio": 0.9750000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 445.0, "completions/mean_length": 1009.5250244140625, "completions/mean_terminated_length": 445.0, "completions/min_length": 445.0, "completions/min_terminated_length": 445.0, "entropy": 0.17103742994368076, "epoch": 0.008532423208191127, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.0052449884824454784, "kl": 0.0009528981681796722, "learning_rate": 8.928327645051195e-06, "loss": 0.0174, "num_tokens": 1689456.0, "reward": 0.05000000074505806, "reward_std": 0.3162277638912201, "rewards/format_reward/mean": 0.02500000037252903, "rewards/format_reward/std": 0.15811389684677124, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.72491455078125, "sampling/importance_sampling_ratio/mean": 0.6370486617088318, "sampling/importance_sampling_ratio/min": 0.0031795057002454996, "sampling/sampling_logp_difference/max": 1.1355410814285278, "sampling/sampling_logp_difference/mean": 0.012314221821725368, "step": 15, "step_time": 74.37915097200312 }, { "clip_ratio/high_max": 0.01408083492424339, "clip_ratio/high_mean": 0.004737149516586214, "clip_ratio/low_mean": 0.001033528671541717, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005770678166300058, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 839.6500244140625, "completions/mean_terminated_length": 635.8947143554688, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "entropy": 0.14754686411470175, "epoch": 0.009101251422070534, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029442468658089638, "kl": 0.0013102454686304554, "learning_rate": 8.92320819112628e-06, "loss": 0.1976, "num_tokens": 1802914.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.8404884338378906, "sampling/importance_sampling_ratio/mean": 0.601033091545105, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5420591831207275, "sampling/sampling_logp_difference/mean": 0.013230669312179089, "step": 16, "step_time": 76.54609363083728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 1024.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 1024.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 1024.0, "completions/min_terminated_length": 0.0, "entropy": 0.1024330435320735, "epoch": 0.009670079635949944, "frac_reward_zero_std": 1.0, "grad_norm": 0.00015938769502099603, "kl": 0.000784165422373917, "learning_rate": 8.918088737201365e-06, "loss": 0.0, "num_tokens": 1911522.0, "reward": 0.0, "reward_std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/gpt_judge_reward/mean": 0.0, "rewards/gpt_judge_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.7587640285491943, "sampling/importance_sampling_ratio/mean": 0.6026450395584106, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6569591760635376, "sampling/sampling_logp_difference/mean": 0.00919740367680788, "step": 17, "step_time": 75.46792920096777 }, { "clip_ratio/high_max": 0.002416750357951969, "clip_ratio/high_mean": 0.0005575340474024415, "clip_ratio/low_mean": 0.0013427734375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019003074994543567, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 991.2250366210938, "completions/mean_terminated_length": 696.25, "completions/min_length": 433.0, "completions/min_terminated_length": 433.0, "entropy": 0.14167999383062124, "epoch": 0.010238907849829351, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.010322650894522667, "kl": 0.0009749966047820635, "learning_rate": 8.912969283276452e-06, "loss": -0.0008, "num_tokens": 2028643.0, "reward": 0.20000000298023224, "reward_std": 0.6076436042785645, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.022934675216675, "sampling/importance_sampling_ratio/mean": 0.43304958939552307, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4921364784240723, "sampling/sampling_logp_difference/mean": 0.01133208628743887, "step": 18, "step_time": 76.97672558040358 }, { "clip_ratio/high_max": 0.004126565560000017, "clip_ratio/high_mean": 0.0008253131200035568, "clip_ratio/low_mean": 0.00046386719623114914, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001289180316234706, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 620.0, "completions/mean_length": 965.4750366210938, "completions/mean_terminated_length": 438.75, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.09422453166916966, "epoch": 0.01080773606370876, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.006529887206852436, "kl": 0.0008340011117979884, "learning_rate": 8.907849829351536e-06, "loss": 0.0342, "num_tokens": 2142190.0, "reward": 0.20000000298023224, "reward_std": 0.6076436042785645, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 1.2849440574645996, "sampling/importance_sampling_ratio/mean": 0.4080590307712555, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8857407569885254, "sampling/sampling_logp_difference/mean": 0.008597495965659618, "step": 19, "step_time": 77.38901927322149 }, { "clip_ratio/high_max": 0.005668755737133324, "clip_ratio/high_mean": 0.0013605085259769112, "clip_ratio/low_mean": 0.0023193359811557457, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003679844521684572, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 996.3500366210938, "completions/mean_terminated_length": 802.7999877929688, "completions/min_length": 602.0, "completions/min_terminated_length": 602.0, "entropy": 0.21093060821294785, "epoch": 0.011376564277588168, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022143688052892685, "kl": 0.0011412416861276142, "learning_rate": 8.902730375426621e-06, "loss": -0.0761, "num_tokens": 2257484.0, "reward": 0.22500000894069672, "reward_std": 0.6196566224098206, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.2323672771453857, "sampling/importance_sampling_ratio/mean": 0.4484466016292572, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.182620644569397, "sampling/sampling_logp_difference/mean": 0.01523987203836441, "step": 20, "step_time": 75.46504782303236 }, { "clip_ratio/high_max": 0.003576493647415191, "clip_ratio/high_mean": 0.0007152987236622721, "clip_ratio/low_mean": 0.00021972656941215973, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009350252930744318, "completions/clipped_ratio": 0.925000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 997.2250366210938, "completions/mean_terminated_length": 667.0, "completions/min_length": 627.0, "completions/min_terminated_length": 627.0, "entropy": 0.07758744806051254, "epoch": 0.011945392491467578, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.00866009946912527, "kl": 0.0007607453517266549, "learning_rate": 8.897610921501707e-06, "loss": 0.061, "num_tokens": 2368901.0, "reward": 0.10000000149011612, "reward_std": 0.3789323568344116, "rewards/format_reward/mean": 0.07500000298023224, "rewards/format_reward/std": 0.26674678921699524, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.5167784690856934, "sampling/importance_sampling_ratio/mean": 0.5438472628593445, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.319961667060852, "sampling/sampling_logp_difference/mean": 0.008079941384494305, "step": 21, "step_time": 76.22774235042743 }, { "clip_ratio/high_max": 0.009105260600335896, "clip_ratio/high_mean": 0.0018210521375294775, "clip_ratio/low_mean": 0.0010986328088620212, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002919684862717986, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 542.0, "completions/mean_length": 955.5499877929688, "completions/mean_terminated_length": 476.3999938964844, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.11108947359025478, "epoch": 0.012514220705346985, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011080523021519184, "kl": 0.0008922049382817931, "learning_rate": 8.892491467576792e-06, "loss": 0.082, "num_tokens": 2470451.0, "reward": 0.25, "reward_std": 0.6698641180992126, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.9361588954925537, "sampling/importance_sampling_ratio/mean": 0.5781596899032593, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.074284553527832, "sampling/sampling_logp_difference/mean": 0.01022855844348669, "step": 22, "step_time": 73.00050340825692 }, { "clip_ratio/high_max": 0.0013736264081671834, "clip_ratio/high_mean": 0.00027472528745420277, "clip_ratio/low_mean": 0.0005126953201397555, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007874206075939583, "completions/clipped_ratio": 0.9750000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 1016.6000366210938, "completions/mean_terminated_length": 728.0, "completions/min_length": 728.0, "completions/min_terminated_length": 728.0, "entropy": 0.07970630656927824, "epoch": 0.013083048919226393, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.01037214882671833, "kl": 0.0009669117062003352, "learning_rate": 8.887372013651876e-06, "loss": -0.0046, "num_tokens": 2581971.0, "reward": 0.05000000074505806, "reward_std": 0.3162277638912201, "rewards/format_reward/mean": 0.02500000037252903, "rewards/format_reward/std": 0.15811389684677124, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.8024604320526123, "sampling/importance_sampling_ratio/mean": 0.5284307599067688, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7622339725494385, "sampling/sampling_logp_difference/mean": 0.008068273775279522, "step": 23, "step_time": 74.71174260112457 }, { "clip_ratio/high_max": 0.0024272822483908385, "clip_ratio/high_mean": 0.0004854564613196999, "clip_ratio/low_mean": 0.0003906250058207661, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.000876081467140466, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 1019.1000366210938, "completions/mean_terminated_length": 975.0, "completions/min_length": 933.0, "completions/min_terminated_length": 933.0, "entropy": 0.0704172682017088, "epoch": 0.013651877133105802, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.010640802793204784, "kl": 0.0007126516429707408, "learning_rate": 8.882252559726963e-06, "loss": 0.0343, "num_tokens": 2674279.0, "reward": 0.20000000298023224, "reward_std": 0.6076436042785645, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.8022377490997314, "sampling/importance_sampling_ratio/mean": 0.60980224609375, "sampling/importance_sampling_ratio/min": 0.0026415488682687283, "sampling/sampling_logp_difference/max": 1.2111663818359375, "sampling/sampling_logp_difference/mean": 0.007420030422508717, "step": 24, "step_time": 70.26737905992195 }, { "clip_ratio/high_max": 0.007883755373768508, "clip_ratio/high_mean": 0.001576751092216, "clip_ratio/low_mean": 0.0018798828241415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003456633916357532, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 968.2750244140625, "completions/mean_terminated_length": 466.75, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.11837159004062414, "epoch": 0.01422070534698521, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01169440895318985, "kl": 0.0009455209219595417, "learning_rate": 8.877133105802049e-06, "loss": 0.087, "num_tokens": 2779554.0, "reward": 0.20000000298023224, "reward_std": 0.6076436042785645, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.6456024646759033, "sampling/importance_sampling_ratio/mean": 0.5794626474380493, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.9508609771728516, "sampling/sampling_logp_difference/mean": 0.00995163805782795, "step": 25, "step_time": 73.90474614570849 }, { "clip_ratio/high_max": 0.003156901802867651, "clip_ratio/high_mean": 0.000631380346021615, "clip_ratio/low_mean": 0.001391601559589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002022981920163147, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 852.0, "completions/mean_length": 921.0250244140625, "completions/mean_terminated_length": 649.5454711914062, "completions/min_length": 488.0, "completions/min_terminated_length": 488.0, "entropy": 0.18475169502198696, "epoch": 0.01478953356086462, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.013364991173148155, "kl": 0.001218523902934976, "learning_rate": 8.872013651877134e-06, "loss": 0.0722, "num_tokens": 2890243.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.156524896621704, "sampling/importance_sampling_ratio/mean": 0.3673838675022125, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3987717628479004, "sampling/sampling_logp_difference/mean": 0.014611673541367054, "step": 26, "step_time": 75.03740565851331 }, { "clip_ratio/high_max": 0.0021576525177806616, "clip_ratio/high_mean": 0.0004315304977353662, "clip_ratio/low_mean": 0.000781250006184564, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012127805039199302, "completions/clipped_ratio": 0.9750000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 869.0, "completions/mean_length": 1020.125, "completions/mean_terminated_length": 869.0, "completions/min_length": 869.0, "completions/min_terminated_length": 869.0, "entropy": 0.12012205366045237, "epoch": 0.015358361774744027, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.003351762657985091, "kl": 0.0008928064780775458, "learning_rate": 8.866894197952218e-06, "loss": 0.0235, "num_tokens": 2998032.0, "reward": 0.05000000074505806, "reward_std": 0.3162277638912201, "rewards/format_reward/mean": 0.02500000037252903, "rewards/format_reward/std": 0.15811389684677124, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.1026132106781006, "sampling/importance_sampling_ratio/mean": 0.5826496481895447, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2092390060424805, "sampling/sampling_logp_difference/mean": 0.010471221059560776, "step": 27, "step_time": 73.86596125294454 }, { "clip_ratio/high_max": 0.0017015706980600953, "clip_ratio/high_mean": 0.00034031414543278515, "clip_ratio/low_mean": 0.0010742187660071068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001414532911439892, "completions/clipped_ratio": 0.9750000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 1022.2750244140625, "completions/mean_terminated_length": 955.0, "completions/min_length": 955.0, "completions/min_terminated_length": 955.0, "entropy": 0.14145651180297136, "epoch": 0.015927189988623434, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.004878494888544083, "kl": 0.0008394639298785478, "learning_rate": 8.861774744027304e-06, "loss": 0.0394, "num_tokens": 3101851.0, "reward": 0.05000000074505806, "reward_std": 0.3162277638912201, "rewards/format_reward/mean": 0.02500000037252903, "rewards/format_reward/std": 0.15811389684677124, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.142756938934326, "sampling/importance_sampling_ratio/mean": 0.46545296907424927, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4363446235656738, "sampling/sampling_logp_difference/mean": 0.0108129708096385, "step": 28, "step_time": 72.24597839755006 }, { "clip_ratio/high_max": 0.0023255813866853714, "clip_ratio/high_mean": 0.00046511628897860646, "clip_ratio/low_mean": 0.0006835937583673513, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011487100473459577, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 993.8250122070312, "completions/mean_terminated_length": 889.888916015625, "completions/min_length": 442.0, "completions/min_terminated_length": 442.0, "entropy": 0.1831701472401619, "epoch": 0.016496018202502846, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.009900779463350773, "kl": 0.0010989965667249635, "learning_rate": 8.856655290102389e-06, "loss": 0.0489, "num_tokens": 3205700.0, "reward": 0.45000001788139343, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.4262590408325195, "sampling/importance_sampling_ratio/mean": 0.45013317465782166, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5517634153366089, "sampling/sampling_logp_difference/mean": 0.014062638394534588, "step": 29, "step_time": 73.646645312896 }, { "clip_ratio/high_max": 0.004985646926797926, "clip_ratio/high_mean": 0.001223373728862498, "clip_ratio/low_mean": 0.0011230468880967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0023464205587515607, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 944.0, "completions/mean_length": 917.2750244140625, "completions/mean_terminated_length": 695.6154174804688, "completions/min_length": 474.0, "completions/min_terminated_length": 474.0, "entropy": 0.16719195991754532, "epoch": 0.017064846416382253, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.02489028498530388, "kl": 0.00126031665422488, "learning_rate": 8.851535836177475e-06, "loss": 0.0236, "num_tokens": 3319279.0, "reward": 0.574999988079071, "reward_std": 0.873762845993042, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.50840425491333, "sampling/importance_sampling_ratio/mean": 0.648220956325531, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4353817701339722, "sampling/sampling_logp_difference/mean": 0.013798254542052746, "step": 30, "step_time": 75.79445971595123 }, { "clip_ratio/high_max": 0.008882139547495171, "clip_ratio/high_mean": 0.0026919536248897202, "clip_ratio/low_mean": 0.00228148112364579, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004973434814019129, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 966.0, "completions/mean_length": 850.8250122070312, "completions/mean_terminated_length": 591.0625, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.11712722666561604, "epoch": 0.01763367463026166, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.011740805581212044, "kl": 0.001137943094363436, "learning_rate": 8.84641638225256e-06, "loss": 0.0551, "num_tokens": 3419232.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.3238186836242676, "sampling/importance_sampling_ratio/mean": 0.4720134139060974, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1610734462738037, "sampling/sampling_logp_difference/mean": 0.010794228874146938, "step": 31, "step_time": 71.05950903170742 }, { "clip_ratio/high_max": 0.0025820721057243645, "clip_ratio/high_mean": 0.000516414424055256, "clip_ratio/low_mean": 0.0010498047067812877, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015662191217415966, "completions/clipped_ratio": 0.949999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 1006.9249877929688, "completions/mean_terminated_length": 682.5, "completions/min_length": 667.0, "completions/min_terminated_length": 667.0, "entropy": 0.05969682335853577, "epoch": 0.01820250284414107, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.03923312574625015, "kl": 0.0007446287927450612, "learning_rate": 8.841296928327644e-06, "loss": -0.0243, "num_tokens": 3526757.0, "reward": 0.10000000149011612, "reward_std": 0.4414428472518921, "rewards/format_reward/mean": 0.05000000074505806, "rewards/format_reward/std": 0.22072142362594604, "rewards/gpt_judge_reward/mean": 0.05000000074505806, "rewards/gpt_judge_reward/std": 0.22072142362594604, "sampling/importance_sampling_ratio/max": 2.844773769378662, "sampling/importance_sampling_ratio/mean": 0.8133581280708313, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0893648862838745, "sampling/sampling_logp_difference/mean": 0.006993570365011692, "step": 32, "step_time": 71.67829333315603 }, { "clip_ratio/high_max": 0.010374244302511215, "clip_ratio/high_mean": 0.002644611377036199, "clip_ratio/low_mean": 0.0021972656104480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0048418769729323685, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 950.0, "completions/mean_length": 934.5250244140625, "completions/mean_terminated_length": 576.625, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.1642945222556591, "epoch": 0.01877133105802048, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02752383053302765, "kl": 0.0011656000060611404, "learning_rate": 8.83617747440273e-06, "loss": 0.0689, "num_tokens": 3637554.0, "reward": 0.3499999940395355, "reward_std": 0.7355444431304932, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 1.8739850521087646, "sampling/importance_sampling_ratio/mean": 0.529108464717865, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2204487323760986, "sampling/sampling_logp_difference/mean": 0.012734482064843178, "step": 33, "step_time": 74.10210283170454 }, { "clip_ratio/high_max": 0.008326007169671357, "clip_ratio/high_mean": 0.0018612798885442317, "clip_ratio/low_mean": 0.0009765624963620212, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002837842417648062, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 951.0750122070312, "completions/mean_terminated_length": 537.8333740234375, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.17239045165479183, "epoch": 0.019340159271899887, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.025384245440363884, "kl": 0.001286828235606663, "learning_rate": 8.831058020477817e-06, "loss": 0.0795, "num_tokens": 3751205.0, "reward": 0.30000001192092896, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.8274123668670654, "sampling/importance_sampling_ratio/mean": 0.6160265803337097, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3881504535675049, "sampling/sampling_logp_difference/mean": 0.013107803650200367, "step": 34, "step_time": 77.67599916202016 }, { "clip_ratio/high_max": 0.0065196703071706, "clip_ratio/high_mean": 0.0013039340410614386, "clip_ratio/low_mean": 0.0004882812572759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017922152983373962, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 982.1749877929688, "completions/mean_terminated_length": 605.75, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.12019234243780375, "epoch": 0.019908987485779295, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.003430150682106614, "kl": 0.001009199462714605, "learning_rate": 8.825938566552902e-06, "loss": 0.0077, "num_tokens": 3866980.0, "reward": 0.15000000596046448, "reward_std": 0.48304590582847595, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.05000000074505806, "rewards/gpt_judge_reward/std": 0.22072142362594604, "sampling/importance_sampling_ratio/max": 1.6896225214004517, "sampling/importance_sampling_ratio/mean": 0.49397316575050354, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2499988079071045, "sampling/sampling_logp_difference/mean": 0.010610023513436317, "step": 35, "step_time": 76.66562029533088 }, { "clip_ratio/high_max": 0.004802940646186471, "clip_ratio/high_mean": 0.0009605881496099755, "clip_ratio/low_mean": 0.0004585578717524186, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001419146021362394, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 957.2000122070312, "completions/mean_terminated_length": 642.2857666015625, "completions/min_length": 506.0, "completions/min_terminated_length": 506.0, "entropy": 0.08560459688305855, "epoch": 0.020477815699658702, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.004863156005740166, "kl": 0.0008142474689520895, "learning_rate": 8.820819112627986e-06, "loss": -0.0116, "num_tokens": 3985076.0, "reward": 0.2750000059604645, "reward_std": 0.6400120258331299, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.133056163787842, "sampling/importance_sampling_ratio/mean": 0.5473709106445312, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2330341339111328, "sampling/sampling_logp_difference/mean": 0.00848267413675785, "step": 36, "step_time": 79.19957674737088 }, { "clip_ratio/high_max": 0.010286001837812364, "clip_ratio/high_mean": 0.0034292986383661628, "clip_ratio/low_mean": 0.0023193359447759576, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.005748634488554671, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 935.75, "completions/mean_terminated_length": 729.8333740234375, "completions/min_length": 429.0, "completions/min_terminated_length": 429.0, "entropy": 0.13211903627961874, "epoch": 0.02104664391353811, "frac_reward_zero_std": 0.0, "grad_norm": 0.015701495110988617, "kl": 0.0009498341460130177, "learning_rate": 8.815699658703072e-06, "loss": 0.0253, "num_tokens": 4095410.0, "reward": 0.4749999940395355, "reward_std": 0.7840558886528015, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.8428475856781006, "sampling/importance_sampling_ratio/mean": 0.360460489988327, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4659042358398438, "sampling/sampling_logp_difference/mean": 0.010883997194468975, "step": 37, "step_time": 74.02763415360823 }, { "clip_ratio/high_max": 0.006937639904208481, "clip_ratio/high_mean": 0.0014941377739887685, "clip_ratio/low_mean": 0.002001953085709829, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034960909106303006, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 976.6000366210938, "completions/mean_terminated_length": 708.0, "completions/min_length": 469.0, "completions/min_terminated_length": 469.0, "entropy": 0.17091690748929977, "epoch": 0.02161547212741752, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014374050311744213, "kl": 0.001174085795355495, "learning_rate": 8.810580204778157e-06, "loss": 0.0792, "num_tokens": 4211306.0, "reward": 0.30000001192092896, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.5722312927246094, "sampling/importance_sampling_ratio/mean": 0.4400199055671692, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.063213586807251, "sampling/sampling_logp_difference/mean": 0.01341412402689457, "step": 38, "step_time": 77.64646037691273 }, { "clip_ratio/high_max": 0.005931456456892192, "clip_ratio/high_mean": 0.0013698448783543427, "clip_ratio/low_mean": 0.0009033203259605216, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022731651915819384, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 968.0, "completions/mean_length": 874.8500366210938, "completions/mean_terminated_length": 626.2667236328125, "completions/min_length": 381.0, "completions/min_terminated_length": 381.0, "entropy": 0.13609227165579796, "epoch": 0.02218430034129693, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.018599428236484528, "kl": 0.0011119388291263022, "learning_rate": 8.805460750853242e-06, "loss": 0.0362, "num_tokens": 4312132.0, "reward": 0.699999988079071, "reward_std": 0.9391758441925049, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.6979517936706543, "sampling/importance_sampling_ratio/mean": 0.5429407954216003, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7088115215301514, "sampling/sampling_logp_difference/mean": 0.012103663757443428, "step": 39, "step_time": 70.23604359291494 }, { "clip_ratio/high_max": 0.009338945324998349, "clip_ratio/high_mean": 0.0021432293142424896, "clip_ratio/low_mean": 0.0012451171951397555, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003388346522115171, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 966.0, "completions/mean_length": 861.4500122070312, "completions/mean_terminated_length": 641.5294189453125, "completions/min_length": 381.0, "completions/min_terminated_length": 381.0, "entropy": 0.13432824332267046, "epoch": 0.022753128555176336, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.023595890030264854, "kl": 0.0011255332719883882, "learning_rate": 8.800341296928328e-06, "loss": 0.0334, "num_tokens": 4420302.0, "reward": 0.824999988079071, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.9149887561798096, "sampling/importance_sampling_ratio/mean": 0.445307195186615, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2072398662567139, "sampling/sampling_logp_difference/mean": 0.011494405567646027, "step": 40, "step_time": 72.462469849037 }, { "clip_ratio/high_max": 0.004065070068463683, "clip_ratio/high_mean": 0.0010304053430445492, "clip_ratio/low_mean": 0.002490234335709829, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035206396714784205, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 969.0, "completions/mean_length": 1003.5250244140625, "completions/mean_terminated_length": 819.25, "completions/min_length": 549.0, "completions/min_terminated_length": 549.0, "entropy": 0.17142491601407528, "epoch": 0.023321956769055744, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.021478308364748955, "kl": 0.0012342362751951441, "learning_rate": 8.795221843003414e-06, "loss": 0.0494, "num_tokens": 4538299.0, "reward": 0.20000000298023224, "reward_std": 0.6076436042785645, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.964184522628784, "sampling/importance_sampling_ratio/mean": 0.5691931247711182, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.255242109298706, "sampling/sampling_logp_difference/mean": 0.012829029932618141, "step": 41, "step_time": 77.85110531863756 }, { "clip_ratio/high_max": 0.009140949696302414, "clip_ratio/high_mean": 0.0021599149040412158, "clip_ratio/low_mean": 0.0015625000232830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037224148982204497, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 829.0, "completions/mean_length": 956.9750366210938, "completions/mean_terminated_length": 577.1666870117188, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.11779612302780151, "epoch": 0.023890784982935155, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011463240720331669, "kl": 0.0010221599441138096, "learning_rate": 8.790102389078499e-06, "loss": 0.0206, "num_tokens": 4650146.0, "reward": 0.32500001788139343, "reward_std": 0.7298577427864075, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.21433687210083, "sampling/importance_sampling_ratio/mean": 0.4894934594631195, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.9888813495635986, "sampling/sampling_logp_difference/mean": 0.010860109701752663, "step": 42, "step_time": 74.80304682813585 }, { "clip_ratio/high_max": 0.002358490601181984, "clip_ratio/high_mean": 0.0004716981202363968, "clip_ratio/low_mean": 0.0015624999978172127, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002034198128967546, "completions/clipped_ratio": 0.949999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 999.2999877929688, "completions/mean_terminated_length": 530.0, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.07819726690649986, "epoch": 0.024459613196814563, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.013423865661025047, "kl": 0.0009411136197741143, "learning_rate": 8.784982935153583e-06, "loss": 0.1116, "num_tokens": 4765742.0, "reward": 0.07500000298023224, "reward_std": 0.34990841150283813, "rewards/format_reward/mean": 0.05000000074505806, "rewards/format_reward/std": 0.22072142362594604, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.9536023139953613, "sampling/importance_sampling_ratio/mean": 0.8705192804336548, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8283374309539795, "sampling/sampling_logp_difference/mean": 0.0078033930622041225, "step": 43, "step_time": 76.90942897298373 }, { "clip_ratio/high_max": 0.006065814755856991, "clip_ratio/high_mean": 0.0014543205325026065, "clip_ratio/low_mean": 0.0011962890785071068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026506096619414166, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 985.5499877929688, "completions/mean_terminated_length": 716.4000244140625, "completions/min_length": 552.0, "completions/min_terminated_length": 552.0, "entropy": 0.1471974803134799, "epoch": 0.02502844141069397, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.012835015542805195, "kl": 0.001094729195756372, "learning_rate": 8.77986348122867e-06, "loss": 0.0233, "num_tokens": 4878612.0, "reward": 0.20000000298023224, "reward_std": 0.5638694167137146, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.189507246017456, "sampling/importance_sampling_ratio/mean": 0.5212094187736511, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1665568351745605, "sampling/sampling_logp_difference/mean": 0.012010958045721054, "step": 44, "step_time": 76.17755987960845 }, { "clip_ratio/high_max": 0.003843033395241946, "clip_ratio/high_mean": 0.0007686066819587722, "clip_ratio/low_mean": 0.0007080078194121597, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001476614501370932, "completions/clipped_ratio": 0.925000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 877.0, "completions/mean_length": 1004.3250122070312, "completions/mean_terminated_length": 761.6666870117188, "completions/min_length": 565.0, "completions/min_terminated_length": 565.0, "entropy": 0.17059603706002235, "epoch": 0.025597269624573378, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.016571803018450737, "kl": 0.0009432979059056379, "learning_rate": 8.774744027303754e-06, "loss": -0.026, "num_tokens": 4988753.0, "reward": 0.07500000298023224, "reward_std": 0.26674678921699524, "rewards/format_reward/mean": 0.07500000298023224, "rewards/format_reward/std": 0.26674678921699524, "rewards/gpt_judge_reward/mean": 0.0, "rewards/gpt_judge_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.380056381225586, "sampling/importance_sampling_ratio/mean": 0.5864718556404114, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.7773518562316895, "sampling/sampling_logp_difference/mean": 0.01264181174337864, "step": 45, "step_time": 75.79605224705301 }, { "clip_ratio/high_max": 0.00838843928067945, "clip_ratio/high_mean": 0.0027083056775154546, "clip_ratio/low_mean": 0.0018554687412688509, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004563774302368984, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 866.3500366210938, "completions/mean_terminated_length": 708.7000122070312, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.1300682295113802, "epoch": 0.026166097838452786, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022942131385207176, "kl": 0.0011382016382412985, "learning_rate": 8.769624573378839e-06, "loss": 0.123, "num_tokens": 5086983.0, "reward": 0.9000000357627869, "reward_std": 0.9554163813591003, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.713703155517578, "sampling/importance_sampling_ratio/mean": 0.6711717247962952, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3761305809020996, "sampling/sampling_logp_difference/mean": 0.010949574410915375, "step": 46, "step_time": 72.11154320812784 }, { "clip_ratio/high_max": 0.009301227575633675, "clip_ratio/high_mean": 0.002786544631817378, "clip_ratio/low_mean": 0.0012971147880307399, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0040836593543644994, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 952.0, "completions/mean_length": 872.6500244140625, "completions/mean_terminated_length": 667.8823852539062, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.1165755270048976, "epoch": 0.026734926052332197, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01689257100224495, "kl": 0.0013219397005741484, "learning_rate": 8.764505119453925e-06, "loss": 0.1164, "num_tokens": 5195665.0, "reward": 0.7250000238418579, "reward_std": 0.905467689037323, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.7319059371948242, "sampling/importance_sampling_ratio/mean": 0.3905399739742279, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3112398386001587, "sampling/sampling_logp_difference/mean": 0.010846675373613834, "step": 47, "step_time": 73.07266397820786 }, { "clip_ratio/high_max": 0.007598450320074335, "clip_ratio/high_mean": 0.0018282279634149745, "clip_ratio/low_mean": 0.001098632830689894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002926860805018805, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 991.375, "completions/mean_terminated_length": 879.0, "completions/min_length": 715.0, "completions/min_terminated_length": 715.0, "entropy": 0.12314384616911411, "epoch": 0.027303754266211604, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.029300978407263756, "kl": 0.0010487614054000005, "learning_rate": 8.759385665529011e-06, "loss": -0.132, "num_tokens": 5296864.0, "reward": 0.4000000059604645, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.289168357849121, "sampling/importance_sampling_ratio/mean": 0.5084510445594788, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4422316551208496, "sampling/sampling_logp_difference/mean": 0.010374578647315502, "step": 48, "step_time": 70.70142806810327 }, { "clip_ratio/high_max": 0.0032810866832733154, "clip_ratio/high_mean": 0.0006562173657584935, "clip_ratio/low_mean": 0.001220703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018769204471027479, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 943.3500366210938, "completions/mean_terminated_length": 701.4000244140625, "completions/min_length": 397.0, "completions/min_terminated_length": 397.0, "entropy": 0.10759358294308186, "epoch": 0.027872582480091012, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01329239085316658, "kl": 0.0011386944315745495, "learning_rate": 8.754266211604096e-06, "loss": 0.1293, "num_tokens": 5406438.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.974863290786743, "sampling/importance_sampling_ratio/mean": 0.7265217304229736, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.113957166671753, "sampling/sampling_logp_difference/mean": 0.009570622816681862, "step": 49, "step_time": 75.25825647078454 }, { "clip_ratio/high_max": 0.008845595410093665, "clip_ratio/high_mean": 0.002511765487724915, "clip_ratio/low_mean": 0.0011962890675931703, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037080545153003186, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 963.0, "completions/mean_length": 946.9500122070312, "completions/mean_terminated_length": 715.7999877929688, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.0760208941064775, "epoch": 0.02844141069397042, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02820521779358387, "kl": 0.0012002439398202114, "learning_rate": 8.74914675767918e-06, "loss": 0.0971, "num_tokens": 5513772.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.4613372087478638, "sampling/importance_sampling_ratio/mean": 0.5233429074287415, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.8754750490188599, "sampling/sampling_logp_difference/mean": 0.007532970514148474, "step": 50, "step_time": 76.53996774810366 }, { "clip_ratio/high_max": 0.00198138365522027, "clip_ratio/high_mean": 0.0003962767223129049, "clip_ratio/low_mean": 0.00043945312791038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008357298502232879, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 833.0, "completions/mean_length": 928.0750122070312, "completions/mean_terminated_length": 640.2999877929688, "completions/min_length": 467.0, "completions/min_terminated_length": 467.0, "entropy": 0.11502928286790848, "epoch": 0.02901023890784983, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.006654971279203892, "kl": 0.001207674853503704, "learning_rate": 8.744027303754267e-06, "loss": 0.0137, "num_tokens": 5619647.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.2548699378967285, "sampling/importance_sampling_ratio/mean": 0.573452353477478, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3335886001586914, "sampling/sampling_logp_difference/mean": 0.010952403768897057, "step": 51, "step_time": 71.74660851899534 }, { "clip_ratio/high_max": 0.010782941593788564, "clip_ratio/high_mean": 0.002469577368174214, "clip_ratio/low_mean": 0.0017578124825377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004227389756124467, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 964.25, "completions/mean_terminated_length": 682.5714721679688, "completions/min_length": 469.0, "completions/min_terminated_length": 469.0, "entropy": 0.1552990609779954, "epoch": 0.02957906712172924, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013072862289845943, "kl": 0.0013921243080403656, "learning_rate": 8.738907849829351e-06, "loss": -0.0145, "num_tokens": 5727537.0, "reward": 0.3499999940395355, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.5081827640533447, "sampling/importance_sampling_ratio/mean": 0.2775183618068695, "sampling/importance_sampling_ratio/min": 0.005369414109736681, "sampling/sampling_logp_difference/max": 1.8631118535995483, "sampling/sampling_logp_difference/mean": 0.012484164908528328, "step": 52, "step_time": 75.28496094280854 }, { "clip_ratio/high_max": 0.00800076505402103, "clip_ratio/high_mean": 0.0023683572144364007, "clip_ratio/low_mean": 0.0023681640450377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004736521223094314, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 927.4249877929688, "completions/mean_terminated_length": 637.7000122070312, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.1504955142736435, "epoch": 0.030147895335608646, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0176699198782444, "kl": 0.0014624616887886077, "learning_rate": 8.733788395904438e-06, "loss": 0.0768, "num_tokens": 5830442.0, "reward": 0.3499999940395355, "reward_std": 0.6621642708778381, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.059370994567871, "sampling/importance_sampling_ratio/mean": 0.4818543791770935, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.257206678390503, "sampling/sampling_logp_difference/mean": 0.012167542241513729, "step": 53, "step_time": 73.71890024701133 }, { "clip_ratio/high_max": 0.006359556311508641, "clip_ratio/high_mean": 0.0019614571283455007, "clip_ratio/low_mean": 0.0019775390392169356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003938996320357546, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 970.7750244140625, "completions/mean_terminated_length": 757.875, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 0.17195724975317717, "epoch": 0.030716723549488054, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.020119229331612587, "kl": 0.0015065562329255044, "learning_rate": 8.728668941979522e-06, "loss": 0.0638, "num_tokens": 5936185.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.8488247394561768, "sampling/importance_sampling_ratio/mean": 0.4965161383152008, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.4422125816345215, "sampling/sampling_logp_difference/mean": 0.013240686617791653, "step": 54, "step_time": 75.2912767783273 }, { "clip_ratio/high_max": 0.009233125310856849, "clip_ratio/high_mean": 0.002821230504196137, "clip_ratio/low_mean": 0.00224698381498456, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005068214290076867, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 886.625, "completions/mean_terminated_length": 680.5625, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "entropy": 0.1850337963551283, "epoch": 0.03128555176336746, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02053913287818432, "kl": 0.0017210400110343471, "learning_rate": 8.723549488054607e-06, "loss": 0.0452, "num_tokens": 6037938.0, "reward": 0.699999988079071, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.67788028717041, "sampling/importance_sampling_ratio/mean": 0.4811352789402008, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.0028538703918457, "sampling/sampling_logp_difference/mean": 0.014419058337807655, "step": 55, "step_time": 72.09071800275706 }, { "clip_ratio/high_max": 0.014985498506575823, "clip_ratio/high_mean": 0.003742939734365791, "clip_ratio/low_mean": 0.0025634765770519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006306416413281113, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 926.6500244140625, "completions/mean_terminated_length": 634.6000366210938, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.16107255592942238, "epoch": 0.03185437997724687, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.033418286591768265, "kl": 0.001942522867466323, "learning_rate": 8.718430034129693e-06, "loss": -0.0489, "num_tokens": 6145388.0, "reward": 0.45000001788139343, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.971580982208252, "sampling/importance_sampling_ratio/mean": 0.4112980365753174, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2061827182769775, "sampling/sampling_logp_difference/mean": 0.013208100572228432, "step": 56, "step_time": 74.89389800769277 }, { "clip_ratio/high_max": 0.011469324817880988, "clip_ratio/high_mean": 0.002777344692731276, "clip_ratio/low_mean": 0.0006835937474534148, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034609384019859135, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 532.0, "completions/mean_length": 904.2750244140625, "completions/mean_terminated_length": 425.375, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.1258073877543211, "epoch": 0.032423208191126277, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.04036172106862068, "kl": 0.001829820277635008, "learning_rate": 8.71331058020478e-06, "loss": 0.0297, "num_tokens": 6241399.0, "reward": 0.4000000059604645, "reward_std": 0.8101914525032043, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.915759801864624, "sampling/importance_sampling_ratio/mean": 0.47188273072242737, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2124872207641602, "sampling/sampling_logp_difference/mean": 0.01075007300823927, "step": 57, "step_time": 70.90278810146265 }, { "clip_ratio/high_max": 0.014231341541744769, "clip_ratio/high_mean": 0.004417782532982528, "clip_ratio/low_mean": 0.0015136718793655746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0059314543614163995, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 914.0, "completions/mean_length": 869.125, "completions/mean_terminated_length": 611.0000610351562, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.1342757772654295, "epoch": 0.03299203640500569, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019734499976038933, "kl": 0.0014974585210438818, "learning_rate": 8.708191126279864e-06, "loss": 0.1011, "num_tokens": 6350756.0, "reward": 0.699999988079071, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.9197003841400146, "sampling/importance_sampling_ratio/mean": 0.4309175908565521, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2774417400360107, "sampling/sampling_logp_difference/mean": 0.011902996338903904, "step": 58, "step_time": 75.43116396199912 }, { "clip_ratio/high_max": 0.007141053269151598, "clip_ratio/high_mean": 0.0020452137978281826, "clip_ratio/low_mean": 0.00138839241117239, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003433606238104403, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 730.7999877929688, "completions/mean_terminated_length": 633.0667114257812, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.0874257069081068, "epoch": 0.0335608646188851, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.035938508808612823, "kl": 0.001984520102269016, "learning_rate": 8.703071672354949e-06, "loss": -0.0229, "num_tokens": 6435724.0, "reward": 1.0750000476837158, "reward_std": 0.7641821503639221, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.8753433227539062, "sampling/importance_sampling_ratio/mean": 0.7185861468315125, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2148513793945312, "sampling/sampling_logp_difference/mean": 0.010518237017095089, "step": 59, "step_time": 69.2917453777045 }, { "clip_ratio/high_max": 0.008166136220097542, "clip_ratio/high_mean": 0.0025269095203839242, "clip_ratio/low_mean": 0.0016113281271827873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004138237592997029, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 893.3500366210938, "completions/mean_terminated_length": 716.5882568359375, "completions/min_length": 525.0, "completions/min_terminated_length": 525.0, "entropy": 0.13659962639212608, "epoch": 0.034129692832764506, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.020405594259500504, "kl": 0.0018359144742134959, "learning_rate": 8.697952218430035e-06, "loss": 0.0617, "num_tokens": 6546074.0, "reward": 0.800000011920929, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.6122987270355225, "sampling/importance_sampling_ratio/mean": 0.5901519656181335, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6750949621200562, "sampling/sampling_logp_difference/mean": 0.012092831544578075, "step": 60, "step_time": 74.1564875531476 }, { "clip_ratio/high_max": 0.0049551646516192704, "clip_ratio/high_mean": 0.001162413442216348, "clip_ratio/low_mean": 0.0014404296671273187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026028431311715394, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 978.4750366210938, "completions/mean_terminated_length": 796.375, "completions/min_length": 493.0, "completions/min_terminated_length": 493.0, "entropy": 0.09533273801207542, "epoch": 0.034698521046643914, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01741650141775608, "kl": 0.0013673457360710017, "learning_rate": 8.69283276450512e-06, "loss": 0.0587, "num_tokens": 6655725.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.5854928493499756, "sampling/importance_sampling_ratio/mean": 0.5298483371734619, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8471744060516357, "sampling/sampling_logp_difference/mean": 0.00930110365152359, "step": 61, "step_time": 75.49561375402845 }, { "clip_ratio/high_max": 0.007770254742354155, "clip_ratio/high_mean": 0.0015540509375568945, "clip_ratio/low_mean": 0.0009033203132275958, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024573712653364055, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 956.7750244140625, "completions/mean_terminated_length": 486.20001220703125, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.15820767357945442, "epoch": 0.03526734926052332, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.013673384673893452, "kl": 0.001689597251242958, "learning_rate": 8.687713310580204e-06, "loss": 0.0049, "num_tokens": 6768428.0, "reward": 0.25, "reward_std": 0.6698641180992126, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.726088285446167, "sampling/importance_sampling_ratio/mean": 0.5027472376823425, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5850489139556885, "sampling/sampling_logp_difference/mean": 0.01231201272457838, "step": 62, "step_time": 75.90831064339727 }, { "clip_ratio/high_max": 0.007193796889623627, "clip_ratio/high_mean": 0.001438759354641661, "clip_ratio/low_mean": 0.0007568359596916707, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021955953197903, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 773.0, "completions/mean_length": 859.2750244140625, "completions/mean_terminated_length": 517.1538696289062, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.13423771876841784, "epoch": 0.03583617747440273, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.012220103293657303, "kl": 0.0024459287524223328, "learning_rate": 8.68259385665529e-06, "loss": 0.0703, "num_tokens": 6870167.0, "reward": 0.6500000357627869, "reward_std": 0.9486833214759827, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.659498453140259, "sampling/importance_sampling_ratio/mean": 0.4965786039829254, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3139631748199463, "sampling/sampling_logp_difference/mean": 0.011642278172075748, "step": 63, "step_time": 72.20073672011495 }, { "clip_ratio/high_max": 0.003786663291975856, "clip_ratio/high_mean": 0.0007573326583951712, "clip_ratio/low_mean": 0.0015625000087311491, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0023198326816782355, "completions/clipped_ratio": 0.949999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 1003.5250244140625, "completions/mean_terminated_length": 614.5, "completions/min_length": 465.0, "completions/min_terminated_length": 465.0, "entropy": 0.13756981398910284, "epoch": 0.03640500568828214, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011182060465216637, "kl": 0.0013507302792277187, "learning_rate": 8.677474402730377e-06, "loss": 0.0712, "num_tokens": 6977884.0, "reward": 0.07500000298023224, "reward_std": 0.34990841150283813, "rewards/format_reward/mean": 0.05000000074505806, "rewards/format_reward/std": 0.22072142362594604, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.989908456802368, "sampling/importance_sampling_ratio/mean": 0.5794622898101807, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6179418563842773, "sampling/sampling_logp_difference/mean": 0.011568374931812286, "step": 64, "step_time": 73.65131128998473 }, { "clip_ratio/high_max": 0.009364361292682588, "clip_ratio/high_mean": 0.0026694623520597816, "clip_ratio/low_mean": 0.003110094625299098, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.005779557075584307, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 908.3500366210938, "completions/mean_terminated_length": 638.5, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.12907469924539328, "epoch": 0.036973833902161544, "frac_reward_zero_std": 0.0, "grad_norm": 0.019789932295680046, "kl": 0.0024094412510748953, "learning_rate": 8.672354948805461e-06, "loss": 0.1129, "num_tokens": 7084754.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.199835777282715, "sampling/importance_sampling_ratio/mean": 0.4184383451938629, "sampling/importance_sampling_ratio/min": 0.008126872591674328, "sampling/sampling_logp_difference/max": 1.6856579780578613, "sampling/sampling_logp_difference/mean": 0.011561497114598751, "step": 65, "step_time": 73.25978544261307 }, { "clip_ratio/high_max": 0.0051912665367126465, "clip_ratio/high_mean": 0.0018836935341823846, "clip_ratio/low_mean": 0.0028076172256987775, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004691310721682385, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 983.25, "completions/mean_terminated_length": 791.1428833007812, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 0.21135104075074196, "epoch": 0.03754266211604096, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015688203275203705, "kl": 0.0022707485913997516, "learning_rate": 8.667235494880546e-06, "loss": 0.108, "num_tokens": 7192404.0, "reward": 0.2750000059604645, "reward_std": 0.6400120258331299, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.2237634658813477, "sampling/importance_sampling_ratio/mean": 0.5377948880195618, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8389368057250977, "sampling/sampling_logp_difference/mean": 0.0155757712200284, "step": 66, "step_time": 74.52662651985884 }, { "clip_ratio/high_max": 0.013689151674043387, "clip_ratio/high_mean": 0.006077874935726868, "clip_ratio/low_mean": 0.0024180125692510046, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008495887217577547, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 725.2250366210938, "completions/mean_terminated_length": 611.8965454101562, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.16113568656146526, "epoch": 0.03811149032992037, "frac_reward_zero_std": 0.0, "grad_norm": 0.02602647989988327, "kl": 0.006233308871742338, "learning_rate": 8.662116040955632e-06, "loss": -0.0704, "num_tokens": 7296685.0, "reward": 1.0, "reward_std": 0.751067578792572, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.691821336746216, "sampling/importance_sampling_ratio/mean": 0.5623012781143188, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6600160598754883, "sampling/sampling_logp_difference/mean": 0.015488318167626858, "step": 67, "step_time": 73.18229877902195 }, { "clip_ratio/high_max": 0.006884204689413309, "clip_ratio/high_mean": 0.0013768409698968753, "clip_ratio/low_mean": 0.0002685546878637979, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016453956577606732, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 987.6749877929688, "completions/mean_terminated_length": 733.4000244140625, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.1385368462651968, "epoch": 0.038680318543799774, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.016229508444666862, "kl": 0.001920244816574268, "learning_rate": 8.656996587030717e-06, "loss": 0.0606, "num_tokens": 7407024.0, "reward": 0.25, "reward_std": 0.6698641180992126, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.0043931007385254, "sampling/importance_sampling_ratio/mean": 0.42324990034103394, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.675123691558838, "sampling/sampling_logp_difference/mean": 0.011624474078416824, "step": 68, "step_time": 77.077720498899 }, { "clip_ratio/high_max": 0.01576306513743475, "clip_ratio/high_mean": 0.005439776330604218, "clip_ratio/low_mean": 0.0017578125116415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007197588973212987, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 867.7000122070312, "completions/mean_terminated_length": 607.2000122070312, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.2042980082333088, "epoch": 0.03924914675767918, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.033113133162260056, "kl": 0.004181441996479407, "learning_rate": 8.651877133105803e-06, "loss": -0.0361, "num_tokens": 7520212.0, "reward": 0.699999988079071, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.9796345233917236, "sampling/importance_sampling_ratio/mean": 0.647070586681366, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3165571689605713, "sampling/sampling_logp_difference/mean": 0.015917779877781868, "step": 69, "step_time": 75.47352553997189 }, { "clip_ratio/high_max": 0.009069638617802411, "clip_ratio/high_mean": 0.002276874889503233, "clip_ratio/low_mean": 0.0013183593673602445, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003595234244130552, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 900.0, "completions/mean_length": 952.6500244140625, "completions/mean_terminated_length": 667.25, "completions/min_length": 486.0, "completions/min_terminated_length": 486.0, "entropy": 0.19168435037136078, "epoch": 0.03981797497155859, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014582900330424309, "kl": 0.002140636308467947, "learning_rate": 8.646757679180888e-06, "loss": 0.043, "num_tokens": 7633686.0, "reward": 0.32500001788139343, "reward_std": 0.6938373446464539, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.3457274436950684, "sampling/importance_sampling_ratio/mean": 0.45163506269454956, "sampling/importance_sampling_ratio/min": 0.006562723778188229, "sampling/sampling_logp_difference/max": 1.450838565826416, "sampling/sampling_logp_difference/mean": 0.014409220777451992, "step": 70, "step_time": 75.10605284525082 }, { "clip_ratio/high_max": 0.015820865897694603, "clip_ratio/high_mean": 0.004484815974137746, "clip_ratio/low_mean": 0.0018066406228172127, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006291456666076556, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 778.7000122070312, "completions/mean_terminated_length": 578.0, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.1520143486559391, "epoch": 0.040386803185438, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0460193045437336, "kl": 0.0067630493140313774, "learning_rate": 8.641638225255972e-06, "loss": 0.2139, "num_tokens": 7742794.0, "reward": 1.100000023841858, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.90793514251709, "sampling/importance_sampling_ratio/mean": 0.5561253428459167, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9747490882873535, "sampling/sampling_logp_difference/mean": 0.015065324492752552, "step": 71, "step_time": 73.09737194119953 }, { "clip_ratio/high_max": 0.010878985456656665, "clip_ratio/high_mean": 0.0026373354921815917, "clip_ratio/low_mean": 0.0020876709822914563, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004725006554508582, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 847.0, "completions/mean_length": 934.5499877929688, "completions/mean_terminated_length": 576.75, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "entropy": 0.14146642945706844, "epoch": 0.040955631399317405, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014643803238868713, "kl": 0.003430943106650375, "learning_rate": 8.636518771331058e-06, "loss": 0.0934, "num_tokens": 7857440.0, "reward": 0.25, "reward_std": 0.5883484482765198, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.977656126022339, "sampling/importance_sampling_ratio/mean": 0.5218013525009155, "sampling/importance_sampling_ratio/min": 0.012296820059418678, "sampling/sampling_logp_difference/max": 1.9072867631912231, "sampling/sampling_logp_difference/mean": 0.012874525971710682, "step": 72, "step_time": 76.99789150361903 }, { "clip_ratio/high_max": 0.0130679389112629, "clip_ratio/high_mean": 0.0050625924341147766, "clip_ratio/low_mean": 0.0011962890675931703, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006258881418034434, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 719.5499877929688, "completions/mean_terminated_length": 536.8800048828125, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.13325622584670782, "epoch": 0.04152445961319681, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02669193409383297, "kl": 0.00582204750389792, "learning_rate": 8.631399317406143e-06, "loss": 0.0229, "num_tokens": 7958526.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.945436477661133, "sampling/importance_sampling_ratio/mean": 0.45194491744041443, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4598755836486816, "sampling/sampling_logp_difference/mean": 0.014169367961585522, "step": 73, "step_time": 71.47119084047154 }, { "clip_ratio/high_max": 0.010837914887815714, "clip_ratio/high_mean": 0.0031536276364931837, "clip_ratio/low_mean": 0.0016845703103172127, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004838197986828163, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 905.0, "completions/mean_length": 801.7250366210938, "completions/mean_terminated_length": 530.0555419921875, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.1520559312775731, "epoch": 0.04209328782707622, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0377485454082489, "kl": 0.009876602940494195, "learning_rate": 8.62627986348123e-06, "loss": 0.0176, "num_tokens": 8067403.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9679524898529053, "sampling/importance_sampling_ratio/mean": 0.7520157098770142, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.382916212081909, "sampling/sampling_logp_difference/mean": 0.014622551389038563, "step": 74, "step_time": 74.89412917592563 }, { "clip_ratio/high_max": 0.007773209479637444, "clip_ratio/high_mean": 0.003162170818541199, "clip_ratio/low_mean": 0.001489257818320766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004651428636861965, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 871.0, "completions/mean_length": 875.2750244140625, "completions/mean_terminated_length": 599.0714721679688, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 0.12090191710740328, "epoch": 0.042662116040955635, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019450249150395393, "kl": 0.005170965509023517, "learning_rate": 8.621160409556314e-06, "loss": 0.0963, "num_tokens": 8179606.0, "reward": 0.5250000357627869, "reward_std": 0.7840558886528015, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.448734998703003, "sampling/importance_sampling_ratio/mean": 0.3615829646587372, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6585638523101807, "sampling/sampling_logp_difference/mean": 0.012661558575928211, "step": 75, "step_time": 74.74369065463543 }, { "clip_ratio/high_max": 0.00942272279644385, "clip_ratio/high_mean": 0.0023536861990578473, "clip_ratio/low_mean": 0.0016743916494306177, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0040280778193846345, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 859.7999877929688, "completions/mean_terminated_length": 586.1333618164062, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.1401950679719448, "epoch": 0.04323094425483504, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.027288220822811127, "kl": 0.0030279166530817747, "learning_rate": 8.6160409556314e-06, "loss": -0.0502, "num_tokens": 8280878.0, "reward": 0.6000000238418579, "reward_std": 0.8412445187568665, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.0586838722229004, "sampling/importance_sampling_ratio/mean": 0.5437813997268677, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.813266396522522, "sampling/sampling_logp_difference/mean": 0.012913691811263561, "step": 76, "step_time": 75.01382250222377 }, { "clip_ratio/high_max": 0.0002604166802484542, "clip_ratio/high_mean": 5.208333459449932e-05, "clip_ratio/low_mean": 0.0002441406304569682, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00029622396505146753, "completions/clipped_ratio": 0.9750000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 480.0, "completions/mean_length": 1010.4000244140625, "completions/mean_terminated_length": 480.0, "completions/min_length": 480.0, "completions/min_terminated_length": 480.0, "entropy": 0.1658063605427742, "epoch": 0.04379977246871445, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.007745477836579084, "kl": 0.0015544502821285278, "learning_rate": 8.610921501706485e-06, "loss": 0.0052, "num_tokens": 8395446.0, "reward": 0.05000000074505806, "reward_std": 0.3162277638912201, "rewards/format_reward/mean": 0.02500000037252903, "rewards/format_reward/std": 0.15811389684677124, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.7031288146972656, "sampling/importance_sampling_ratio/mean": 0.35099488496780396, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.069437026977539, "sampling/sampling_logp_difference/mean": 0.013172432780265808, "step": 77, "step_time": 74.57546394178644 }, { "clip_ratio/high_max": 0.0034352182410657406, "clip_ratio/high_mean": 0.0006870436482131481, "clip_ratio/low_mean": 0.0011474609509605216, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018345045900787227, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 853.0, "completions/mean_length": 902.2250366210938, "completions/mean_terminated_length": 581.1818237304688, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.1611709799617529, "epoch": 0.04436860068259386, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.029362931847572327, "kl": 0.00334017799468711, "learning_rate": 8.60580204778157e-06, "loss": -0.0847, "num_tokens": 8500951.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.842637062072754, "sampling/importance_sampling_ratio/mean": 0.45201754570007324, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.60872220993042, "sampling/sampling_logp_difference/mean": 0.013971748761832714, "step": 78, "step_time": 72.21709154709242 }, { "clip_ratio/high_max": 0.010284771880833432, "clip_ratio/high_mean": 0.002870143682230264, "clip_ratio/low_mean": 0.0003970894031226635, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003267233114456758, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 841.0, "completions/mean_length": 893.5499877929688, "completions/mean_terminated_length": 651.2857666015625, "completions/min_length": 385.0, "completions/min_terminated_length": 385.0, "entropy": 0.08070797752588987, "epoch": 0.044937428896473265, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011853338219225407, "kl": 0.0035675636609084904, "learning_rate": 8.600682593856656e-06, "loss": 0.0095, "num_tokens": 8607429.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.5210301876068115, "sampling/importance_sampling_ratio/mean": 0.34721165895462036, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.189746379852295, "sampling/sampling_logp_difference/mean": 0.009654094465076923, "step": 79, "step_time": 76.11036899825558 }, { "clip_ratio/high_max": 0.011046327534131706, "clip_ratio/high_mean": 0.0037683799273509067, "clip_ratio/low_mean": 0.001900143472084892, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005668523401254788, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 880.0, "completions/mean_length": 812.2999877929688, "completions/mean_terminated_length": 578.3157958984375, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.15539886057376862, "epoch": 0.04550625711035267, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03269258514046669, "kl": 0.00674741790862754, "learning_rate": 8.595563139931742e-06, "loss": -0.0541, "num_tokens": 8715513.0, "reward": 0.875, "reward_std": 0.9657599329948425, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.8168246746063232, "sampling/importance_sampling_ratio/mean": 0.5344434976577759, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1919126510620117, "sampling/sampling_logp_difference/mean": 0.014205075800418854, "step": 80, "step_time": 74.03295717644505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 801.0, "completions/mean_length": 934.4000244140625, "completions/mean_terminated_length": 576.0, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.16259955428540707, "epoch": 0.04607508532423208, "frac_reward_zero_std": 1.0, "grad_norm": 0.0005485263536684215, "kl": 0.002358295358135365, "learning_rate": 8.590443686006826e-06, "loss": 0.0, "num_tokens": 8829009.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.9723703861236572, "sampling/importance_sampling_ratio/mean": 0.44172483682632446, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7333168983459473, "sampling/sampling_logp_difference/mean": 0.013364200480282307, "step": 81, "step_time": 74.5156460290309 }, { "clip_ratio/high_max": 0.005640367220621556, "clip_ratio/high_mean": 0.0011280734615866095, "clip_ratio/low_mean": 0.0004150390705035534, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015431125320901629, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 692.8250122070312, "completions/mean_terminated_length": 550.8928833007812, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.14401062205433846, "epoch": 0.04664391353811149, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.01523918379098177, "kl": 0.01250736805377528, "learning_rate": 8.585324232081911e-06, "loss": -0.0024, "num_tokens": 8928162.0, "reward": 1.399999976158142, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.699999988079071, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.5749785900115967, "sampling/importance_sampling_ratio/mean": 0.47354739904403687, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.060943126678467, "sampling/sampling_logp_difference/mean": 0.01571323350071907, "step": 82, "step_time": 71.78364959685132 }, { "clip_ratio/high_max": 0.017821727320551872, "clip_ratio/high_mean": 0.00611133046913892, "clip_ratio/low_mean": 0.0012207031213620212, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00733203359413892, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 771.5, "completions/mean_terminated_length": 462.8888854980469, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.09898968134075403, "epoch": 0.047212741751990896, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.031160252168774605, "kl": 0.007156802050303668, "learning_rate": 8.580204778156997e-06, "loss": 0.0655, "num_tokens": 9031942.0, "reward": 0.9000000357627869, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.3578763008117676, "sampling/importance_sampling_ratio/mean": 0.6438344120979309, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.554001569747925, "sampling/sampling_logp_difference/mean": 0.011702680960297585, "step": 83, "step_time": 72.14547403715551 }, { "clip_ratio/high_max": 0.004413749906234443, "clip_ratio/high_mean": 0.0008827499550534412, "clip_ratio/low_mean": 0.0013916015668655746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002274351500091143, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 907.0750122070312, "completions/mean_terminated_length": 634.25, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.12574004288762808, "epoch": 0.04778156996587031, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.012090803124010563, "kl": 0.023367796646198258, "learning_rate": 8.575085324232082e-06, "loss": 0.0156, "num_tokens": 9146121.0, "reward": 0.574999988079071, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.5344042778015137, "sampling/importance_sampling_ratio/mean": 0.4006417393684387, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.152587890625, "sampling/sampling_logp_difference/mean": 0.012333780527114868, "step": 84, "step_time": 76.2357634587679 }, { "clip_ratio/high_max": 0.006812525214627385, "clip_ratio/high_mean": 0.0013625050341943279, "clip_ratio/low_mean": 0.0021972656395519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003559770731953904, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 742.2250366210938, "completions/mean_terminated_length": 460.45001220703125, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.13360122591257095, "epoch": 0.04835039817974972, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04617539420723915, "kl": 0.014773094386328012, "learning_rate": 8.569965870307167e-06, "loss": -0.0821, "num_tokens": 9255746.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.5712568759918213, "sampling/importance_sampling_ratio/mean": 0.6135910749435425, "sampling/importance_sampling_ratio/min": 0.004517144989222288, "sampling/sampling_logp_difference/max": 3.5447683334350586, "sampling/sampling_logp_difference/mean": 0.014400511980056763, "step": 85, "step_time": 71.54334071069025 }, { "clip_ratio/high_max": 0.012630293378606439, "clip_ratio/high_mean": 0.004556444386253133, "clip_ratio/low_mean": 0.0011718749883584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005728319258196279, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 968.0, "completions/mean_length": 765.4249877929688, "completions/mean_terminated_length": 593.0416870117188, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.11918606050312519, "epoch": 0.048919226393629126, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0270072054117918, "kl": 0.02126422803848982, "learning_rate": 8.564846416382253e-06, "loss": -0.0111, "num_tokens": 9354939.0, "reward": 0.9750000238418579, "reward_std": 0.8911962509155273, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.8343729972839355, "sampling/importance_sampling_ratio/mean": 0.4185475707054138, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7989795207977295, "sampling/sampling_logp_difference/mean": 0.013340420089662075, "step": 86, "step_time": 71.76309289759956 }, { "clip_ratio/high_max": 0.006219631031854078, "clip_ratio/high_mean": 0.001655647618463263, "clip_ratio/low_mean": 0.0019042969142901711, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035599445982370526, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 790.0, "completions/mean_length": 962.7999877929688, "completions/mean_terminated_length": 674.2857666015625, "completions/min_length": 396.0, "completions/min_terminated_length": 396.0, "entropy": 0.09498355351388454, "epoch": 0.04948805460750853, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.046367477625608444, "kl": 0.016779397090431303, "learning_rate": 8.559726962457337e-06, "loss": 0.0154, "num_tokens": 9464059.0, "reward": 0.3499999940395355, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.3773880004882812, "sampling/importance_sampling_ratio/mean": 0.4593939185142517, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.002096176147461, "sampling/sampling_logp_difference/mean": 0.009907037951052189, "step": 87, "step_time": 75.03502867207862 }, { "clip_ratio/high_max": 0.014756889548152685, "clip_ratio/high_mean": 0.004561583758913912, "clip_ratio/low_mean": 0.001562499990541255, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006124083622125909, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 950.0, "completions/mean_length": 862.6749877929688, "completions/mean_terminated_length": 563.0714721679688, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.1388624757528305, "epoch": 0.05005688282138794, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02318429760634899, "kl": 0.005612056062091142, "learning_rate": 8.554607508532424e-06, "loss": 0.0366, "num_tokens": 9570926.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.567396879196167, "sampling/importance_sampling_ratio/mean": 0.5362919569015503, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3378777503967285, "sampling/sampling_logp_difference/mean": 0.012583275325596333, "step": 88, "step_time": 72.74736550613306 }, { "clip_ratio/high_max": 0.009556181787047535, "clip_ratio/high_mean": 0.002012450946494937, "clip_ratio/low_mean": 0.000732421884094947, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002744872821494937, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 811.2000122070312, "completions/mean_terminated_length": 456.5333557128906, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.12181808892637491, "epoch": 0.05062571103526735, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.02022223174571991, "kl": 0.005355272529413924, "learning_rate": 8.549488054607508e-06, "loss": 0.0019, "num_tokens": 9681470.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.653890609741211, "sampling/importance_sampling_ratio/mean": 0.5403605699539185, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4502921104431152, "sampling/sampling_logp_difference/mean": 0.011911186389625072, "step": 89, "step_time": 74.3019296603743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 971.0, "completions/mean_length": 837.9000244140625, "completions/mean_terminated_length": 558.75, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.18197974376380444, "epoch": 0.051194539249146756, "frac_reward_zero_std": 1.0, "grad_norm": 0.0009909061482176185, "kl": 0.004412956448504701, "learning_rate": 8.544368600682595e-06, "loss": 0.0, "num_tokens": 9789922.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.7200090885162354, "sampling/importance_sampling_ratio/mean": 0.5378904938697815, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.002243757247925, "sampling/sampling_logp_difference/mean": 0.015483266673982143, "step": 90, "step_time": 73.19513740483671 }, { "clip_ratio/high_max": 0.00694402534281835, "clip_ratio/high_mean": 0.001951236426975811, "clip_ratio/low_mean": 0.0023437500203726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004294986429158598, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 955.0250244140625, "completions/mean_terminated_length": 811.769287109375, "completions/min_length": 547.0, "completions/min_terminated_length": 547.0, "entropy": 0.0691849971190095, "epoch": 0.051763367463026164, "frac_reward_zero_std": 0.0, "grad_norm": 0.041586365550756454, "kl": 0.003651357052149251, "learning_rate": 8.539249146757679e-06, "loss": 0.0199, "num_tokens": 9894643.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.9014856815338135, "sampling/importance_sampling_ratio/mean": 0.7652291059494019, "sampling/importance_sampling_ratio/min": 0.03507331758737564, "sampling/sampling_logp_difference/max": 1.3005621433258057, "sampling/sampling_logp_difference/mean": 0.007932348176836967, "step": 91, "step_time": 72.0050096756313 }, { "clip_ratio/high_max": 0.009234649973222986, "clip_ratio/high_mean": 0.003133830505248625, "clip_ratio/low_mean": 0.001759387960191816, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004893218487268314, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 975.0, "completions/mean_length": 905.1500244140625, "completions/mean_terminated_length": 773.7894897460938, "completions/min_length": 506.0, "completions/min_terminated_length": 506.0, "entropy": 0.17223883420228958, "epoch": 0.05233219567690557, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022690365090966225, "kl": 0.005547856126213446, "learning_rate": 8.534129692832765e-06, "loss": -0.0522, "num_tokens": 10006265.0, "reward": 0.800000011920929, "reward_std": 0.8828856945037842, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.6667628288269043, "sampling/importance_sampling_ratio/mean": 0.4219807982444763, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3157589435577393, "sampling/sampling_logp_difference/mean": 0.014549748972058296, "step": 92, "step_time": 77.17534232581966 }, { "clip_ratio/high_max": 0.013182326394598931, "clip_ratio/high_mean": 0.0032755951688159257, "clip_ratio/low_mean": 0.0020507812514551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00532637641299516, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 979.0, "completions/mean_length": 961.6000366210938, "completions/mean_terminated_length": 797.0909423828125, "completions/min_length": 673.0, "completions/min_terminated_length": 673.0, "entropy": 0.14761792682111263, "epoch": 0.052901023890784986, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013954122550785542, "kl": 0.003716899751452729, "learning_rate": 8.52901023890785e-06, "loss": 0.0292, "num_tokens": 10116937.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.8706943988800049, "sampling/importance_sampling_ratio/mean": 0.4233672618865967, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9855165481567383, "sampling/sampling_logp_difference/mean": 0.013258546590805054, "step": 93, "step_time": 76.13322376552969 }, { "clip_ratio/high_max": 0.011573953001061454, "clip_ratio/high_mean": 0.0027068108174717054, "clip_ratio/low_mean": 0.002734208115725778, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005441018904093653, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 861.0499877929688, "completions/mean_terminated_length": 740.6087036132812, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.1606254242360592, "epoch": 0.053469852104664393, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.031232895329594612, "kl": 0.013791658653644845, "learning_rate": 8.523890784982935e-06, "loss": -0.0447, "num_tokens": 10235755.0, "reward": 1.0, "reward_std": 0.9336995482444763, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.995962142944336, "sampling/importance_sampling_ratio/mean": 0.44651660323143005, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0146896839141846, "sampling/sampling_logp_difference/mean": 0.015268792398273945, "step": 94, "step_time": 78.46294116810896 }, { "clip_ratio/high_max": 0.006616766506340355, "clip_ratio/high_mean": 0.0015115590940695256, "clip_ratio/low_mean": 0.001660156271100277, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003171715317876078, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 797.0, "completions/mean_length": 950.125, "completions/mean_terminated_length": 531.5, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.10622281953692436, "epoch": 0.0540386803185438, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.043855421245098114, "kl": 0.01089002433582209, "learning_rate": 8.518771331058021e-06, "loss": -0.0559, "num_tokens": 10335440.0, "reward": 0.30000001192092896, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.9147729873657227, "sampling/importance_sampling_ratio/mean": 0.4625505805015564, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1522216796875, "sampling/sampling_logp_difference/mean": 0.010541893541812897, "step": 95, "step_time": 71.72506438218988 }, { "clip_ratio/high_max": 0.013072981382720172, "clip_ratio/high_mean": 0.004011068202089518, "clip_ratio/low_mean": 0.00021972656031721272, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004230794758768752, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 794.5, "completions/mean_terminated_length": 606.727294921875, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.18240170739591122, "epoch": 0.05460750853242321, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.008006080985069275, "kl": 0.0031489167595282197, "learning_rate": 8.513651877133107e-06, "loss": -0.0511, "num_tokens": 10442244.0, "reward": 1.100000023841858, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.926337718963623, "sampling/importance_sampling_ratio/mean": 0.43959808349609375, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.938244342803955, "sampling/sampling_logp_difference/mean": 0.016760213300585747, "step": 96, "step_time": 73.35895009431988 }, { "clip_ratio/high_max": 0.0063873870531097054, "clip_ratio/high_mean": 0.0015713109605712816, "clip_ratio/low_mean": 0.0017089843695430318, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003280295350123197, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 981.1500244140625, "completions/mean_terminated_length": 809.75, "completions/min_length": 547.0, "completions/min_terminated_length": 547.0, "entropy": 0.16032664570957422, "epoch": 0.055176336746302616, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025432728230953217, "kl": 0.002819768022163771, "learning_rate": 8.508532423208192e-06, "loss": 0.1268, "num_tokens": 10559362.0, "reward": 0.3499999940395355, "reward_std": 0.7355444431304932, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.6175379753112793, "sampling/importance_sampling_ratio/mean": 0.5075535178184509, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8847439289093018, "sampling/sampling_logp_difference/mean": 0.012356515042483807, "step": 97, "step_time": 79.51543463696726 }, { "clip_ratio/high_max": 0.00852296408265829, "clip_ratio/high_mean": 0.0033131325617432594, "clip_ratio/low_mean": 0.0024414062354480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00575453884084709, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 963.7250366210938, "completions/mean_terminated_length": 804.8181762695312, "completions/min_length": 582.0, "completions/min_terminated_length": 582.0, "entropy": 0.17544593289494514, "epoch": 0.055745164960182024, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022784480825066566, "kl": 0.003742927889106795, "learning_rate": 8.503412969283276e-06, "loss": -0.0512, "num_tokens": 10673223.0, "reward": 0.4749999940395355, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.4068143367767334, "sampling/importance_sampling_ratio/mean": 0.4178498387336731, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8730363845825195, "sampling/sampling_logp_difference/mean": 0.014541208744049072, "step": 98, "step_time": 75.05439668241888 }, { "clip_ratio/high_max": 0.009824234992265701, "clip_ratio/high_mean": 0.0025133125600405037, "clip_ratio/low_mean": 0.0013427734593278728, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003856085939332843, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 914.0, "completions/mean_length": 955.1500244140625, "completions/mean_terminated_length": 565.0, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.167284375987947, "epoch": 0.05631399317406143, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.017651455476880074, "kl": 0.0032371823908761144, "learning_rate": 8.498293515358363e-06, "loss": 0.0419, "num_tokens": 10776733.0, "reward": 0.30000001192092896, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.45981502532959, "sampling/importance_sampling_ratio/mean": 0.5236701369285583, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.783353328704834, "sampling/sampling_logp_difference/mean": 0.013143899850547314, "step": 99, "step_time": 73.13465452799574 }, { "clip_ratio/high_max": 0.011745293973945081, "clip_ratio/high_mean": 0.0028042606718372554, "clip_ratio/low_mean": 0.0019140673430229072, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004718327953014523, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 789.0, "completions/mean_length": 844.7999877929688, "completions/mean_terminated_length": 602.3529663085938, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.09174861386418343, "epoch": 0.05688282138794084, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.021279212087392807, "kl": 0.0034787816111929715, "learning_rate": 8.493174061433447e-06, "loss": -0.0365, "num_tokens": 10878173.0, "reward": 0.675000011920929, "reward_std": 0.8589648008346558, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.9688948392868042, "sampling/importance_sampling_ratio/mean": 0.4304417669773102, "sampling/importance_sampling_ratio/min": 0.023627063259482384, "sampling/sampling_logp_difference/max": 1.731273889541626, "sampling/sampling_logp_difference/mean": 0.009928295388817787, "step": 100, "step_time": 70.64611528883688 }, { "clip_ratio/high_max": 0.0134280389174819, "clip_ratio/high_mean": 0.0034984425583388656, "clip_ratio/low_mean": 0.0015869140570430318, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005085356620838866, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 979.0, "completions/mean_length": 916.4000244140625, "completions/mean_terminated_length": 716.5714721679688, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.12640205025672913, "epoch": 0.05745164960182025, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015293617732822895, "kl": 0.0042163261387031525, "learning_rate": 8.488054607508532e-06, "loss": -0.0443, "num_tokens": 10982157.0, "reward": 0.574999988079071, "reward_std": 0.8439072966575623, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 1.6253949403762817, "sampling/importance_sampling_ratio/mean": 0.3697587847709656, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.001859664916992, "sampling/sampling_logp_difference/mean": 0.011553041636943817, "step": 101, "step_time": 73.35838235518895 }, { "clip_ratio/high_max": 0.008599286142271012, "clip_ratio/high_mean": 0.00278005184372887, "clip_ratio/low_mean": 0.0020868998835794628, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004866951785515994, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 845.0, "completions/mean_length": 840.4500122070312, "completions/mean_terminated_length": 534.5333862304688, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.1538808811455965, "epoch": 0.05802047781569966, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.017032664269208908, "kl": 0.0046294986677821726, "learning_rate": 8.482935153583618e-06, "loss": -0.0525, "num_tokens": 11081047.0, "reward": 0.699999988079071, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.2277607917785645, "sampling/importance_sampling_ratio/mean": 0.385714590549469, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.963287115097046, "sampling/sampling_logp_difference/mean": 0.014664341695606709, "step": 102, "step_time": 71.89942176104523 }, { "clip_ratio/high_max": 0.013323338003829122, "clip_ratio/high_mean": 0.006134370371000841, "clip_ratio/low_mean": 0.0025193887413479388, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.00865375908324495, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 813.7999877929688, "completions/mean_terminated_length": 603.6000366210938, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.15902652963995934, "epoch": 0.05858930602957907, "frac_reward_zero_std": 0.0, "grad_norm": 0.038674838840961456, "kl": 0.005490804847795516, "learning_rate": 8.477815699658704e-06, "loss": -0.1605, "num_tokens": 11188623.0, "reward": 0.824999988079071, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.201070547103882, "sampling/importance_sampling_ratio/mean": 0.5489605069160461, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.018251895904541, "sampling/sampling_logp_difference/mean": 0.01498500443994999, "step": 103, "step_time": 78.00893329293467 }, { "clip_ratio/high_max": 0.010593510582111776, "clip_ratio/high_mean": 0.0022839914163341746, "clip_ratio/low_mean": 0.0017481885661254637, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00403217991697602, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 809.0750122070312, "completions/mean_terminated_length": 614.6190795898438, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.09117983654141426, "epoch": 0.05915813424345848, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025639953091740608, "kl": 0.003077028988627717, "learning_rate": 8.472696245733789e-06, "loss": 0.0846, "num_tokens": 11286538.0, "reward": 0.9000000357627869, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.408249616622925, "sampling/importance_sampling_ratio/mean": 0.5349486470222473, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.881699800491333, "sampling/sampling_logp_difference/mean": 0.010197334922850132, "step": 104, "step_time": 72.6925075687468 }, { "clip_ratio/high_max": 0.011958206538110971, "clip_ratio/high_mean": 0.002858508101780899, "clip_ratio/low_mean": 0.0023681641032453626, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052266720740590245, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 916.0, "completions/mean_length": 909.0750122070312, "completions/mean_terminated_length": 606.0909423828125, "completions/min_length": 406.0, "completions/min_terminated_length": 406.0, "entropy": 0.15157040301710367, "epoch": 0.059726962457337884, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01108057051897049, "kl": 0.0034865716588683426, "learning_rate": 8.467576791808874e-06, "loss": 0.0209, "num_tokens": 11395349.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.1573445796966553, "sampling/importance_sampling_ratio/mean": 0.36532527208328247, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7365195751190186, "sampling/sampling_logp_difference/mean": 0.013120085000991821, "step": 105, "step_time": 73.16146662505344 }, { "clip_ratio/high_max": 0.011711384227965027, "clip_ratio/high_mean": 0.003227679611882195, "clip_ratio/low_mean": 0.0009765625163709046, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0042042421555379406, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 928.1500244140625, "completions/mean_terminated_length": 675.45458984375, "completions/min_length": 435.0, "completions/min_terminated_length": 435.0, "entropy": 0.18845930136740208, "epoch": 0.06029579067121729, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01179258618503809, "kl": 0.004601355554768816, "learning_rate": 8.46245733788396e-06, "loss": -0.0119, "num_tokens": 11505867.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.601705551147461, "sampling/importance_sampling_ratio/mean": 0.311835378408432, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7030386924743652, "sampling/sampling_logp_difference/mean": 0.015587218105793, "step": 106, "step_time": 74.96224544150755 }, { "clip_ratio/high_max": 0.013188832555897534, "clip_ratio/high_mean": 0.005952947132755071, "clip_ratio/low_mean": 0.001855468777648639, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.007808415801264346, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 838.9750366210938, "completions/mean_terminated_length": 653.9500122070312, "completions/min_length": 432.0, "completions/min_terminated_length": 432.0, "entropy": 0.1648693773895502, "epoch": 0.0608646188850967, "frac_reward_zero_std": 0.0, "grad_norm": 0.03517100214958191, "kl": 0.005171040626009926, "learning_rate": 8.457337883959044e-06, "loss": 0.0606, "num_tokens": 11613418.0, "reward": 0.800000011920929, "reward_std": 0.8828856945037842, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.9357478618621826, "sampling/importance_sampling_ratio/mean": 0.4310055673122406, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8375015258789062, "sampling/sampling_logp_difference/mean": 0.014719448052346706, "step": 107, "step_time": 74.57726585981436 }, { "clip_ratio/high_max": 0.009673524415120482, "clip_ratio/high_mean": 0.003304939193185419, "clip_ratio/low_mean": 0.002221679693320766, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.0055266187991946936, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 900.0, "completions/mean_length": 869.5250244140625, "completions/mean_terminated_length": 582.6428833007812, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.12122836988419294, "epoch": 0.06143344709897611, "frac_reward_zero_std": 0.0, "grad_norm": 0.02030852995812893, "kl": 0.0034620941733010113, "learning_rate": 8.452218430034129e-06, "loss": 0.0167, "num_tokens": 11720343.0, "reward": 0.6000000238418579, "reward_std": 0.8711913824081421, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.9623903036117554, "sampling/importance_sampling_ratio/mean": 0.3888551592826843, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.002932548522949, "sampling/sampling_logp_difference/mean": 0.010822822339832783, "step": 108, "step_time": 72.30471157492138 }, { "clip_ratio/high_max": 0.0082309499848634, "clip_ratio/high_mean": 0.0021589933458017185, "clip_ratio/low_mean": 0.0005613816101686098, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0027203749195905402, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 925.0499877929688, "completions/mean_terminated_length": 664.1818237304688, "completions/min_length": 480.0, "completions/min_terminated_length": 480.0, "entropy": 0.12898522149771452, "epoch": 0.062002275312855515, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.012178567238152027, "kl": 0.0034252671466674656, "learning_rate": 8.447098976109215e-06, "loss": 0.0556, "num_tokens": 11832641.0, "reward": 0.42500001192092896, "reward_std": 0.747217059135437, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.461618423461914, "sampling/importance_sampling_ratio/mean": 0.5793363451957703, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5264344215393066, "sampling/sampling_logp_difference/mean": 0.01131803821772337, "step": 109, "step_time": 74.69027991872281 }, { "clip_ratio/high_max": 0.014900552108883858, "clip_ratio/high_mean": 0.00478820368880406, "clip_ratio/low_mean": 0.0016845703139551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006472774141002446, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 900.0, "completions/mean_length": 886.7999877929688, "completions/mean_terminated_length": 632.0, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.16954136826097965, "epoch": 0.06257110352673492, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01858765073120594, "kl": 0.005585588107351214, "learning_rate": 8.4419795221843e-06, "loss": 0.0532, "num_tokens": 11932377.0, "reward": 0.5, "reward_std": 0.751067578792572, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.8379058837890625, "sampling/importance_sampling_ratio/mean": 0.5371590852737427, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2447280883789062, "sampling/sampling_logp_difference/mean": 0.015109089203178883, "step": 110, "step_time": 71.1543813047465 }, { "clip_ratio/high_max": 0.011895206989720464, "clip_ratio/high_mean": 0.003760809951927513, "clip_ratio/low_mean": 0.0014648437245341484, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005225653731031343, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 867.5250244140625, "completions/mean_terminated_length": 606.7333374023438, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.17421313375234604, "epoch": 0.06313993174061433, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018492890521883965, "kl": 0.003633470361819491, "learning_rate": 8.436860068259386e-06, "loss": 0.1214, "num_tokens": 12033878.0, "reward": 0.699999988079071, "reward_std": 0.9391758441925049, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.516911506652832, "sampling/importance_sampling_ratio/mean": 0.32806211709976196, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7348921298980713, "sampling/sampling_logp_difference/mean": 0.014396597631275654, "step": 111, "step_time": 71.91515404521488 }, { "clip_ratio/high_max": 0.004129487671889365, "clip_ratio/high_mean": 0.0008258975503849797, "clip_ratio/low_mean": 0.0002685546951397555, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010944522455247352, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 961.0, "completions/mean_length": 820.4249877929688, "completions/mean_terminated_length": 616.8500366210938, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "entropy": 0.21473521180450916, "epoch": 0.06370875995449374, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.015464344061911106, "kl": 0.007898721611127257, "learning_rate": 8.43174061433447e-06, "loss": 0.0626, "num_tokens": 12142167.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.9767760038375854, "sampling/importance_sampling_ratio/mean": 0.5089600682258606, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1791388988494873, "sampling/sampling_logp_difference/mean": 0.0174409132450819, "step": 112, "step_time": 74.41767234914005 }, { "clip_ratio/high_max": 0.009129090700298548, "clip_ratio/high_mean": 0.0021620517363771796, "clip_ratio/low_mean": 0.0026611328357830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0048231845430564135, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 962.0, "completions/mean_length": 932.0, "completions/mean_terminated_length": 564.0, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.13342582434415817, "epoch": 0.06427758816837315, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01350091490894556, "kl": 0.003939765272662044, "learning_rate": 8.426621160409557e-06, "loss": 0.0316, "num_tokens": 12251839.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.7992624044418335, "sampling/importance_sampling_ratio/mean": 0.4554496705532074, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.836223840713501, "sampling/sampling_logp_difference/mean": 0.012315313331782818, "step": 113, "step_time": 73.99971942044795 }, { "clip_ratio/high_max": 0.014383373432792723, "clip_ratio/high_mean": 0.006183911114931107, "clip_ratio/low_mean": 0.0007881360197643517, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006972047151066363, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 822.4750366210938, "completions/mean_terminated_length": 657.5909423828125, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.14251934178173542, "epoch": 0.06484641638225255, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019790709018707275, "kl": 0.004802591254701838, "learning_rate": 8.421501706484642e-06, "loss": -0.0244, "num_tokens": 12359122.0, "reward": 1.0750000476837158, "reward_std": 0.9971111416816711, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.2830634117126465, "sampling/importance_sampling_ratio/mean": 0.4770945608615875, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7300035953521729, "sampling/sampling_logp_difference/mean": 0.013533580116927624, "step": 114, "step_time": 74.26236652280204 }, { "clip_ratio/high_max": 0.011876774136908352, "clip_ratio/high_mean": 0.003414129067095928, "clip_ratio/low_mean": 0.0031250000320142135, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.00653912901179865, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 898.0, "completions/mean_length": 918.5250244140625, "completions/mean_terminated_length": 602.1000366210938, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.15892077051103115, "epoch": 0.06541524459613197, "frac_reward_zero_std": 0.0, "grad_norm": 0.04198980703949928, "kl": 0.006540248723467812, "learning_rate": 8.416382252559728e-06, "loss": 0.0449, "num_tokens": 12472431.0, "reward": 0.5, "reward_std": 0.8770580887794495, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.377540111541748, "sampling/importance_sampling_ratio/mean": 0.48988834023475647, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9321110248565674, "sampling/sampling_logp_difference/mean": 0.01346580684185028, "step": 115, "step_time": 75.19360857899301 }, { "clip_ratio/high_max": 0.007618615869432688, "clip_ratio/high_mean": 0.0016955444880295545, "clip_ratio/low_mean": 0.001489257821958745, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003184802219038829, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 952.1749877929688, "completions/mean_terminated_length": 449.3999938964844, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.11159433424472809, "epoch": 0.06598407281001138, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01582500711083412, "kl": 0.003889715750119649, "learning_rate": 8.411262798634812e-06, "loss": 0.0553, "num_tokens": 12591430.0, "reward": 0.25, "reward_std": 0.6698641180992126, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.2626254558563232, "sampling/importance_sampling_ratio/mean": 0.41180533170700073, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0029263496398926, "sampling/sampling_logp_difference/mean": 0.010688725858926773, "step": 116, "step_time": 77.43597052828409 }, { "clip_ratio/high_max": 0.011657011578790843, "clip_ratio/high_mean": 0.002667705630301498, "clip_ratio/low_mean": 0.0007812500043655746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034489556128391996, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 905.0, "completions/mean_length": 932.7000122070312, "completions/mean_terminated_length": 618.2222290039062, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.17373285442590714, "epoch": 0.06655290102389079, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.022081047296524048, "kl": 0.012481460609706119, "learning_rate": 8.406143344709897e-06, "loss": 0.0177, "num_tokens": 12703234.0, "reward": 0.45000001788139343, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.991266965866089, "sampling/importance_sampling_ratio/mean": 0.6139956712722778, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9798858165740967, "sampling/sampling_logp_difference/mean": 0.01447319146245718, "step": 117, "step_time": 76.22983027691953 }, { "clip_ratio/high_max": 0.010318122600438073, "clip_ratio/high_mean": 0.003071504244871903, "clip_ratio/low_mean": 0.0012695312398136593, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0043410354846855626, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 884.4750366210938, "completions/mean_terminated_length": 516.6363525390625, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.0686292601749301, "epoch": 0.0671217292377702, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01735004410147667, "kl": 0.004186082980595529, "learning_rate": 8.401023890784983e-06, "loss": 0.037, "num_tokens": 12799949.0, "reward": 0.4749999940395355, "reward_std": 0.8161038756370544, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.865742564201355, "sampling/importance_sampling_ratio/mean": 0.36065635085105896, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.248664617538452, "sampling/sampling_logp_difference/mean": 0.008199699223041534, "step": 118, "step_time": 69.99730652431026 }, { "clip_ratio/high_max": 0.015826103277504444, "clip_ratio/high_mean": 0.0044117409124737605, "clip_ratio/low_mean": 0.00031738281177240424, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004729123698780313, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 853.5, "completions/mean_terminated_length": 499.3846435546875, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.12977880705147982, "epoch": 0.0676905574516496, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01980062760412693, "kl": 0.005927898368099704, "learning_rate": 8.39590443686007e-06, "loss": 0.065, "num_tokens": 12902849.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.25425124168396, "sampling/importance_sampling_ratio/mean": 0.43813610076904297, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.004059314727783, "sampling/sampling_logp_difference/mean": 0.013126181438565254, "step": 119, "step_time": 72.93810252379626 }, { "clip_ratio/high_max": 0.002151462947949767, "clip_ratio/high_mean": 0.0006051177624613047, "clip_ratio/low_mean": 0.0012695312507275958, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018746490131889004, "completions/clipped_ratio": 0.949999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 581.0, "completions/mean_length": 998.0499877929688, "completions/mean_terminated_length": 505.0, "completions/min_length": 429.0, "completions/min_terminated_length": 429.0, "entropy": 0.09748353529721498, "epoch": 0.06825938566552901, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.008804898709058762, "kl": 0.00282504377537407, "learning_rate": 8.390784982935154e-06, "loss": 0.0583, "num_tokens": 13014307.0, "reward": 0.07500000298023224, "reward_std": 0.34990841150283813, "rewards/format_reward/mean": 0.05000000074505806, "rewards/format_reward/std": 0.22072142362594604, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.1703200340270996, "sampling/importance_sampling_ratio/mean": 0.7173534631729126, "sampling/importance_sampling_ratio/min": 0.01739930734038353, "sampling/sampling_logp_difference/max": 1.4336323738098145, "sampling/sampling_logp_difference/mean": 0.009203609079122543, "step": 120, "step_time": 74.00226533180103 }, { "clip_ratio/high_max": 0.011361516604665667, "clip_ratio/high_mean": 0.004273198610462714, "clip_ratio/low_mean": 0.0021073232346680015, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0063805218087509274, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 852.2250366210938, "completions/mean_terminated_length": 662.368408203125, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.09692245628684759, "epoch": 0.06882821387940842, "frac_reward_zero_std": 0.0, "grad_norm": 0.02402345836162567, "kl": 0.007181533612310886, "learning_rate": 8.385665529010239e-06, "loss": 0.0962, "num_tokens": 13111340.0, "reward": 0.9000000357627869, "reward_std": 0.9818872213363647, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.6396327018737793, "sampling/importance_sampling_ratio/mean": 0.35686272382736206, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5922324657440186, "sampling/sampling_logp_difference/mean": 0.010885079391300678, "step": 121, "step_time": 69.60915733897127 }, { "clip_ratio/high_max": 0.013505872280802578, "clip_ratio/high_mean": 0.003100008558249101, "clip_ratio/low_mean": 0.0013183593764551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004418367781909183, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 921.2999877929688, "completions/mean_terminated_length": 613.2000122070312, "completions/min_length": 431.0, "completions/min_terminated_length": 431.0, "entropy": 0.09297279827296734, "epoch": 0.06939704209328783, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03384871035814285, "kl": 0.0036699235206469893, "learning_rate": 8.380546075085325e-06, "loss": -0.0677, "num_tokens": 13226240.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.6495134830474854, "sampling/importance_sampling_ratio/mean": 0.4943910241127014, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5519317388534546, "sampling/sampling_logp_difference/mean": 0.009672857820987701, "step": 122, "step_time": 76.42600136063993 }, { "clip_ratio/high_max": 0.014945981267374009, "clip_ratio/high_mean": 0.004554338651360013, "clip_ratio/low_mean": 0.003247070257202722, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.007801408995874226, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 956.0, "completions/mean_length": 936.8500366210938, "completions/mean_terminated_length": 733.5, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.1758321514353156, "epoch": 0.06996587030716724, "frac_reward_zero_std": 0.0, "grad_norm": 0.016380349174141884, "kl": 0.005206352157983929, "learning_rate": 8.37542662116041e-06, "loss": 0.0486, "num_tokens": 13340242.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.362375497817993, "sampling/importance_sampling_ratio/mean": 0.27972912788391113, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2148749828338623, "sampling/sampling_logp_difference/mean": 0.015610146336257458, "step": 123, "step_time": 74.94311772217043 }, { "clip_ratio/high_max": 0.005477041006088257, "clip_ratio/high_mean": 0.0010954082245007157, "clip_ratio/low_mean": 0.0005126953183207661, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016081035428214818, "completions/clipped_ratio": 0.925000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 620.0, "completions/mean_length": 993.4000244140625, "completions/mean_terminated_length": 616.0, "completions/min_length": 614.0, "completions/min_terminated_length": 614.0, "entropy": 0.08008455811068416, "epoch": 0.07053469852104664, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.017426030710339546, "kl": 0.002523925300920382, "learning_rate": 8.370307167235494e-06, "loss": -0.0273, "num_tokens": 13452826.0, "reward": 0.15000000596046448, "reward_std": 0.5334935784339905, "rewards/format_reward/mean": 0.07500000298023224, "rewards/format_reward/std": 0.26674678921699524, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.5811641216278076, "sampling/importance_sampling_ratio/mean": 0.535443127155304, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.752950668334961, "sampling/sampling_logp_difference/mean": 0.008687569759786129, "step": 124, "step_time": 74.28864676156081 }, { "clip_ratio/high_max": 0.013496433501131833, "clip_ratio/high_mean": 0.0041890915017575026, "clip_ratio/low_mean": 0.0008300781337311491, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005019169591832906, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 816.0, "completions/mean_length": 851.4500122070312, "completions/mean_terminated_length": 531.0, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.12622248101979494, "epoch": 0.07110352673492605, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014320221729576588, "kl": 0.009837780002271757, "learning_rate": 8.36518771331058e-06, "loss": -0.0183, "num_tokens": 13563524.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.163367986679077, "sampling/importance_sampling_ratio/mean": 0.4499531686306, "sampling/importance_sampling_ratio/min": 0.0007062375079840422, "sampling/sampling_logp_difference/max": 4.259117603302002, "sampling/sampling_logp_difference/mean": 0.012926208786666393, "step": 125, "step_time": 75.95989911886863 }, { "clip_ratio/high_max": 0.00915785925462842, "clip_ratio/high_mean": 0.0030422754207393155, "clip_ratio/low_mean": 0.0030251969292294234, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.006067472219001502, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 864.4249877929688, "completions/mean_terminated_length": 720.047607421875, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.13864151947200298, "epoch": 0.07167235494880546, "frac_reward_zero_std": 0.0, "grad_norm": 0.028163889423012733, "kl": 0.0036711313005071133, "learning_rate": 8.360068259385665e-06, "loss": 0.0664, "num_tokens": 13669589.0, "reward": 0.7250000238418579, "reward_std": 0.7840558886528015, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.9722001552581787, "sampling/importance_sampling_ratio/mean": 0.41362810134887695, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.735426664352417, "sampling/sampling_logp_difference/mean": 0.01317219715565443, "step": 126, "step_time": 71.93327288120054 }, { "clip_ratio/high_max": 0.015944984974339604, "clip_ratio/high_mean": 0.006308252632152289, "clip_ratio/low_mean": 0.0010253906511934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00733364321058616, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 812.5, "completions/mean_terminated_length": 639.45458984375, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.13155433908104897, "epoch": 0.07224118316268487, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027380086481571198, "kl": 0.005176590580958873, "learning_rate": 8.354948805460751e-06, "loss": -0.0307, "num_tokens": 13775153.0, "reward": 0.925000011920929, "reward_std": 0.9167249798774719, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.967728614807129, "sampling/importance_sampling_ratio/mean": 0.621616542339325, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7933204174041748, "sampling/sampling_logp_difference/mean": 0.0130969462916255, "step": 127, "step_time": 74.0449547462631 }, { "clip_ratio/high_max": 0.0019455594010651112, "clip_ratio/high_mean": 0.0003891118831234053, "clip_ratio/low_mean": 0.0011474609491415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015365728322649375, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 889.2250366210938, "completions/mean_terminated_length": 484.8999938964844, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.11160438880324364, "epoch": 0.07281001137656427, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.004931486677378416, "kl": 0.0032872977026272565, "learning_rate": 8.349829351535836e-06, "loss": 0.0089, "num_tokens": 13887562.0, "reward": 0.45000001788139343, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.98514461517334, "sampling/importance_sampling_ratio/mean": 0.6083822250366211, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.462456464767456, "sampling/sampling_logp_difference/mean": 0.012096012942492962, "step": 128, "step_time": 73.72738744784147 }, { "clip_ratio/high_max": 0.009123939904384315, "clip_ratio/high_mean": 0.0024531737726647407, "clip_ratio/low_mean": 0.0014648437645519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003918017551768571, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 829.0, "completions/mean_length": 930.7999877929688, "completions/mean_terminated_length": 558.0, "completions/min_length": 419.0, "completions/min_terminated_length": 419.0, "entropy": 0.12786822486668825, "epoch": 0.07337883959044368, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018114550039172173, "kl": 0.004743865836644545, "learning_rate": 8.344709897610922e-06, "loss": 0.0326, "num_tokens": 14003242.0, "reward": 0.3499999940395355, "reward_std": 0.7355444431304932, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.379060745239258, "sampling/importance_sampling_ratio/mean": 0.5455690622329712, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.827012538909912, "sampling/sampling_logp_difference/mean": 0.01273426879197359, "step": 129, "step_time": 77.05406327592209 }, { "clip_ratio/high_max": 0.010112276795553043, "clip_ratio/high_mean": 0.0030762165260966867, "clip_ratio/low_mean": 0.002252821606816724, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005329038016498089, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 935.0, "completions/mean_length": 838.5499877929688, "completions/mean_terminated_length": 587.6470336914062, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.15690152160823345, "epoch": 0.07394766780432309, "frac_reward_zero_std": 0.0, "grad_norm": 0.0572752021253109, "kl": 0.005555239535169676, "learning_rate": 8.339590443686007e-06, "loss": -0.0489, "num_tokens": 14111552.0, "reward": 0.6500000357627869, "reward_std": 0.8638019561767578, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.7367303371429443, "sampling/importance_sampling_ratio/mean": 0.39977067708969116, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.386035680770874, "sampling/sampling_logp_difference/mean": 0.01454716082662344, "step": 130, "step_time": 72.33593844622374 }, { "clip_ratio/high_max": 0.007640996889676899, "clip_ratio/high_mean": 0.0019773050444200635, "clip_ratio/low_mean": 0.0013183593528083293, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003295664384495467, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 946.1000366210938, "completions/mean_terminated_length": 634.5, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "entropy": 0.10273983143270016, "epoch": 0.0745164960182025, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.036508407443761826, "kl": 0.0032906595442909747, "learning_rate": 8.334470989761093e-06, "loss": 0.0224, "num_tokens": 14216924.0, "reward": 0.2750000059604645, "reward_std": 0.5986095070838928, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509575605392456, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.8851120471954346, "sampling/importance_sampling_ratio/mean": 0.49015313386917114, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.499868869781494, "sampling/sampling_logp_difference/mean": 0.010269886814057827, "step": 131, "step_time": 73.13062466145493 }, { "clip_ratio/high_max": 0.006821446295361966, "clip_ratio/high_mean": 0.0019394041446503252, "clip_ratio/low_mean": 0.0007324218822759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002671826019650325, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 935.5499877929688, "completions/mean_terminated_length": 771.2857666015625, "completions/min_length": 597.0, "completions/min_terminated_length": 597.0, "entropy": 0.0627323086373508, "epoch": 0.07508532423208192, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01104680448770523, "kl": 0.002583794586826116, "learning_rate": 8.329351535836178e-06, "loss": -0.0299, "num_tokens": 14318818.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.3515801429748535, "sampling/importance_sampling_ratio/mean": 0.46342068910598755, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.25911808013916, "sampling/sampling_logp_difference/mean": 0.007694849278777838, "step": 132, "step_time": 74.38200067169964 }, { "clip_ratio/high_max": 0.014974933117628098, "clip_ratio/high_mean": 0.004437939467607066, "clip_ratio/low_mean": 0.0011888976587215438, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005626837286399677, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 747.7000122070312, "completions/mean_terminated_length": 543.478271484375, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "entropy": 0.13816399313509464, "epoch": 0.07565415244596133, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.012707872316241264, "kl": 0.019869364565238357, "learning_rate": 8.324232081911262e-06, "loss": 0.0057, "num_tokens": 14425694.0, "reward": 1.100000023841858, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.827659249305725, "sampling/importance_sampling_ratio/mean": 0.30849260091781616, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9016542434692383, "sampling/sampling_logp_difference/mean": 0.014964669942855835, "step": 133, "step_time": 73.47540420223959 }, { "clip_ratio/high_max": 0.003958461282309145, "clip_ratio/high_mean": 0.0007916922586446162, "clip_ratio/low_mean": 0.0007568359542347025, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015485281619476154, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 994.6000366210938, "completions/mean_terminated_length": 828.0, "completions/min_length": 595.0, "completions/min_terminated_length": 595.0, "entropy": 0.1818972434848547, "epoch": 0.07622298065984073, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01166547555476427, "kl": 0.004760545183671638, "learning_rate": 8.319112627986349e-06, "loss": 0.0232, "num_tokens": 14538174.0, "reward": 0.30000001192092896, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.150421380996704, "sampling/importance_sampling_ratio/mean": 0.38081979751586914, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2656192779541016, "sampling/sampling_logp_difference/mean": 0.014283216558396816, "step": 134, "step_time": 75.35788241401315 }, { "clip_ratio/high_max": 0.0068662636913359165, "clip_ratio/high_mean": 0.0016090419376268983, "clip_ratio/low_mean": 0.0007568359433207661, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002365877866395749, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 923.5499877929688, "completions/mean_terminated_length": 756.1333618164062, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "entropy": 0.1178486505523324, "epoch": 0.07679180887372014, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.015288537368178368, "kl": 0.0038886585825821385, "learning_rate": 8.313993174061435e-06, "loss": 0.016, "num_tokens": 14650412.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.9198685884475708, "sampling/importance_sampling_ratio/mean": 0.3798072338104248, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.915398120880127, "sampling/sampling_logp_difference/mean": 0.011684348806738853, "step": 135, "step_time": 75.51711966539733 }, { "clip_ratio/high_max": 0.011858439771458507, "clip_ratio/high_mean": 0.004574221355142072, "clip_ratio/low_mean": 0.0022949218546273187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006869143166113645, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 990.0, "completions/mean_length": 876.8250122070312, "completions/mean_terminated_length": 696.9444580078125, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.1324706682935357, "epoch": 0.07736063708759955, "frac_reward_zero_std": 0.0, "grad_norm": 0.02494916133582592, "kl": 0.006926257046870887, "learning_rate": 8.30887372013652e-06, "loss": 0.0787, "num_tokens": 14760333.0, "reward": 0.9000000357627869, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.8295716047286987, "sampling/importance_sampling_ratio/mean": 0.3474868834018707, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2234268188476562, "sampling/sampling_logp_difference/mean": 0.013276922516524792, "step": 136, "step_time": 75.96225862484425 }, { "clip_ratio/high_max": 0.015940985293127596, "clip_ratio/high_mean": 0.004112143498787191, "clip_ratio/low_mean": 0.0023193359083961695, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006431479414459318, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.0, "completions/mean_length": 768.7999877929688, "completions/mean_terminated_length": 513.6000366210938, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.1656041881069541, "epoch": 0.07792946530147896, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.028649095445871353, "kl": 0.00711393408710137, "learning_rate": 8.303754266211604e-06, "loss": 0.0215, "num_tokens": 14867717.0, "reward": 0.9000000357627869, "reward_std": 0.9554163813591003, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.036808490753174, "sampling/importance_sampling_ratio/mean": 0.43610721826553345, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5856493711471558, "sampling/sampling_logp_difference/mean": 0.015228547155857086, "step": 137, "step_time": 74.22743572364561 }, { "clip_ratio/high_max": 0.010235922527499497, "clip_ratio/high_mean": 0.0020471845200518146, "clip_ratio/low_mean": 0.0017578124716237653, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003804997046245262, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 846.4750366210938, "completions/mean_terminated_length": 550.6000366210938, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.15762652456760406, "epoch": 0.07849829351535836, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011339273303747177, "kl": 0.005052521242760122, "learning_rate": 8.29863481228669e-06, "loss": 0.0593, "num_tokens": 14974248.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.2178876399993896, "sampling/importance_sampling_ratio/mean": 0.37380504608154297, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8260420560836792, "sampling/sampling_logp_difference/mean": 0.014706184156239033, "step": 138, "step_time": 73.08944758074358 }, { "clip_ratio/high_max": 0.0050616313819773495, "clip_ratio/high_mean": 0.0012056964478688315, "clip_ratio/low_mean": 0.001220703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002426399543765001, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 885.2000122070312, "completions/mean_terminated_length": 627.4285888671875, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.09925319720059633, "epoch": 0.07906712172923777, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.010505679063498974, "kl": 0.010846077464520931, "learning_rate": 8.293515358361775e-06, "loss": 0.0327, "num_tokens": 15081704.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.6971896886825562, "sampling/importance_sampling_ratio/mean": 0.32302218675613403, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.5091896057128906, "sampling/sampling_logp_difference/mean": 0.011093774810433388, "step": 139, "step_time": 71.51554726273753 }, { "clip_ratio/high_max": 0.013859556929674, "clip_ratio/high_mean": 0.006368168484186754, "clip_ratio/low_mean": 0.001978511754714418, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008346680144313723, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 990.0, "completions/mean_length": 828.3500366210938, "completions/mean_terminated_length": 683.7391357421875, "completions/min_length": 448.0, "completions/min_terminated_length": 448.0, "entropy": 0.17119844630360603, "epoch": 0.07963594994311718, "frac_reward_zero_std": 0.0, "grad_norm": 0.012750668451189995, "kl": 0.0057855635532177985, "learning_rate": 8.28839590443686e-06, "loss": -0.0161, "num_tokens": 15191102.0, "reward": 0.9750000238418579, "reward_std": 0.9195176959037781, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.4961389899253845, "sampling/importance_sampling_ratio/max": 1.3168729543685913, "sampling/importance_sampling_ratio/mean": 0.25675415992736816, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0278849601745605, "sampling/sampling_logp_difference/mean": 0.0161649901419878, "step": 140, "step_time": 71.93034434318542 }, { "clip_ratio/high_max": 0.012062319554388523, "clip_ratio/high_mean": 0.003158888459438458, "clip_ratio/low_mean": 0.0028564452950377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006015333812683821, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 925.0, "completions/mean_length": 869.5250244140625, "completions/mean_terminated_length": 509.0833435058594, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.10241872444748878, "epoch": 0.08020477815699659, "frac_reward_zero_std": 0.0, "grad_norm": 0.05226343125104904, "kl": 0.00717149896081537, "learning_rate": 8.283276450511946e-06, "loss": -0.0408, "num_tokens": 15303147.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.7434489727020264, "sampling/importance_sampling_ratio/mean": 0.4357604384422302, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.573392629623413, "sampling/sampling_logp_difference/mean": 0.011520286090672016, "step": 141, "step_time": 74.85523909679614 }, { "clip_ratio/high_max": 0.015003564301878214, "clip_ratio/high_mean": 0.005025997990742326, "clip_ratio/low_mean": 0.002270507766297669, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007296505791600794, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 936.2000122070312, "completions/mean_terminated_length": 773.1428833007812, "completions/min_length": 525.0, "completions/min_terminated_length": 525.0, "entropy": 0.17500020004808903, "epoch": 0.080773606370876, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03870352730154991, "kl": 0.006383856351021677, "learning_rate": 8.278156996587032e-06, "loss": -0.1424, "num_tokens": 15416467.0, "reward": 0.6500000357627869, "reward_std": 0.9212588667869568, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.5968480110168457, "sampling/importance_sampling_ratio/mean": 0.5293641090393066, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.97455894947052, "sampling/sampling_logp_difference/mean": 0.015925703570246696, "step": 142, "step_time": 76.65908540575765 }, { "clip_ratio/high_max": 0.006941302621271461, "clip_ratio/high_mean": 0.0018448416813043877, "clip_ratio/low_mean": 0.0027136976168549154, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004558539221761748, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 936.0, "completions/mean_length": 944.2750244140625, "completions/mean_terminated_length": 705.1000366210938, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.16230329684913158, "epoch": 0.0813424345847554, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02642403356730938, "kl": 0.006155750481411815, "learning_rate": 8.273037542662117e-06, "loss": 0.0604, "num_tokens": 15530270.0, "reward": 0.45000001788139343, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509575605392456, "sampling/importance_sampling_ratio/max": 2.7069013118743896, "sampling/importance_sampling_ratio/mean": 0.505943775177002, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 9.589000701904297, "sampling/sampling_logp_difference/mean": 0.014230380766093731, "step": 143, "step_time": 76.22271271795034 }, { "clip_ratio/high_max": 0.01714627246838063, "clip_ratio/high_mean": 0.006882258225232363, "clip_ratio/low_mean": 0.001806640619179234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008688898989930749, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 807.5499877929688, "completions/mean_terminated_length": 630.45458984375, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.13783432357013226, "epoch": 0.08191126279863481, "frac_reward_zero_std": 0.0, "grad_norm": 0.06377486139535904, "kl": 0.019986844330560416, "learning_rate": 8.267918088737201e-06, "loss": -0.0946, "num_tokens": 15638548.0, "reward": 0.949999988079071, "reward_std": 0.9323254823684692, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.1094303131103516, "sampling/importance_sampling_ratio/mean": 0.4084153175354004, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.8690567016601562, "sampling/sampling_logp_difference/mean": 0.013284862041473389, "step": 144, "step_time": 74.06445566983894 }, { "clip_ratio/high_max": 0.011357092531397939, "clip_ratio/high_mean": 0.003238721750676632, "clip_ratio/low_mean": 0.0018364698844379745, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005075191555079073, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 913.2999877929688, "completions/mean_terminated_length": 707.7142944335938, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.17476987279951572, "epoch": 0.08248009101251422, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024653421714901924, "kl": 0.008313450613059103, "learning_rate": 8.262798634812287e-06, "loss": -0.0099, "num_tokens": 15754120.0, "reward": 0.6000000238418579, "reward_std": 0.8711913824081421, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.34468936920166, "sampling/importance_sampling_ratio/mean": 0.5067055821418762, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7988479137420654, "sampling/sampling_logp_difference/mean": 0.015884490683674812, "step": 145, "step_time": 76.8804827556014 }, { "clip_ratio/high_max": 0.010367433889769018, "clip_ratio/high_mean": 0.0032441825314890593, "clip_ratio/low_mean": 0.001123046884458745, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004367229412309825, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 811.9750366210938, "completions/mean_terminated_length": 599.9500122070312, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "entropy": 0.11456043645739555, "epoch": 0.08304891922639362, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.026392707601189613, "kl": 0.011633296264335513, "learning_rate": 8.257679180887372e-06, "loss": -0.0771, "num_tokens": 15859927.0, "reward": 0.925000011920929, "reward_std": 0.9710554480552673, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.7132811546325684, "sampling/importance_sampling_ratio/mean": 0.4426373839378357, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6585594415664673, "sampling/sampling_logp_difference/mean": 0.01247416716068983, "step": 146, "step_time": 72.6694424781017 }, { "clip_ratio/high_max": 0.009622227284125984, "clip_ratio/high_mean": 0.003391265985555947, "clip_ratio/low_mean": 0.0028796957776648924, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062709617195650935, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 888.0, "completions/mean_length": 871.4000244140625, "completions/mean_terminated_length": 588.0, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.17334025911986828, "epoch": 0.08361774744027303, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026913102716207504, "kl": 0.007601552817504853, "learning_rate": 8.252559726962457e-06, "loss": 0.0813, "num_tokens": 15969655.0, "reward": 0.625, "reward_std": 0.9250780344009399, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.1666600704193115, "sampling/importance_sampling_ratio/mean": 0.4815073013305664, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2229928970336914, "sampling/sampling_logp_difference/mean": 0.015839675441384315, "step": 147, "step_time": 72.38568009692244 }, { "clip_ratio/high_max": 0.011429675098042935, "clip_ratio/high_mean": 0.00282023980980739, "clip_ratio/low_mean": 0.001921106995723676, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004741346725495532, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 977.0, "completions/mean_length": 795.2000122070312, "completions/mean_terminated_length": 566.4000244140625, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.14337299205362797, "epoch": 0.08418657565415244, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014121958054602146, "kl": 0.005917770846281201, "learning_rate": 8.247440273037543e-06, "loss": 0.0447, "num_tokens": 16068487.0, "reward": 0.925000011920929, "reward_std": 0.9971111416816711, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.814669609069824, "sampling/importance_sampling_ratio/mean": 0.4853098392486572, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.441208600997925, "sampling/sampling_logp_difference/mean": 0.013092108070850372, "step": 148, "step_time": 75.86098661390133 }, { "clip_ratio/high_max": 0.011227028619032353, "clip_ratio/high_mean": 0.002245405747089535, "clip_ratio/low_mean": 0.001440429681679234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036858353996649384, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 968.0250244140625, "completions/mean_terminated_length": 704.1428833007812, "completions/min_length": 549.0, "completions/min_terminated_length": 549.0, "entropy": 0.11988251376897097, "epoch": 0.08475540386803185, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.017783701419830322, "kl": 0.005295415467116982, "learning_rate": 8.242320819112628e-06, "loss": 0.0686, "num_tokens": 16182912.0, "reward": 0.3499999940395355, "reward_std": 0.7696152329444885, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.7714900970458984, "sampling/importance_sampling_ratio/mean": 0.5797764658927917, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2163915634155273, "sampling/sampling_logp_difference/mean": 0.011501013301312923, "step": 149, "step_time": 76.94078157888725 }, { "clip_ratio/high_max": 0.0029498684452846646, "clip_ratio/high_mean": 0.0005899736861465499, "clip_ratio/low_mean": 0.00019531249745341484, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007852861835999647, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 906.2999877929688, "completions/mean_terminated_length": 631.6666870117188, "completions/min_length": 470.0, "completions/min_terminated_length": 470.0, "entropy": 0.10421490017324686, "epoch": 0.08532423208191127, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.013150430284440517, "kl": 0.0035896069603040814, "learning_rate": 8.237201365187714e-06, "loss": 0.0433, "num_tokens": 16288812.0, "reward": 0.5, "reward_std": 0.8164966106414795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.287506580352783, "sampling/importance_sampling_ratio/mean": 0.5306326746940613, "sampling/importance_sampling_ratio/min": 0.015217040665447712, "sampling/sampling_logp_difference/max": 2.1750001907348633, "sampling/sampling_logp_difference/mean": 0.010942061431705952, "step": 150, "step_time": 74.56024661846459 }, { "clip_ratio/high_max": 0.00901435175910592, "clip_ratio/high_mean": 0.002824574359692633, "clip_ratio/low_mean": 0.0009765625200088834, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038011368305888027, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 828.0, "completions/mean_length": 800.9000244140625, "completions/mean_terminated_length": 554.3157958984375, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.10101533774286509, "epoch": 0.08589306029579068, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03140062838792801, "kl": 0.004800628055818379, "learning_rate": 8.232081911262798e-06, "loss": 0.0737, "num_tokens": 16391816.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.8603687286376953, "sampling/importance_sampling_ratio/mean": 0.4009135365486145, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5369584560394287, "sampling/sampling_logp_difference/mean": 0.01161827053874731, "step": 151, "step_time": 76.17849678173661 }, { "clip_ratio/high_max": 0.012942250352352858, "clip_ratio/high_mean": 0.004266030533472076, "clip_ratio/low_mean": 0.0016882066302059684, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005954237247351557, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 861.3250122070312, "completions/mean_terminated_length": 752.875, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "entropy": 0.12018211372196674, "epoch": 0.08646188850967008, "frac_reward_zero_std": 0.0, "grad_norm": 0.03487887233495712, "kl": 0.004471964988624677, "learning_rate": 8.226962457337885e-06, "loss": 0.154, "num_tokens": 16498533.0, "reward": 0.9000000357627869, "reward_std": 0.8711913824081421, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.239692211151123, "sampling/importance_sampling_ratio/mean": 0.37870195508003235, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2027485370635986, "sampling/sampling_logp_difference/mean": 0.012532171793282032, "step": 152, "step_time": 72.66064892220311 }, { "clip_ratio/high_max": 0.012794336245860904, "clip_ratio/high_mean": 0.004231733466440346, "clip_ratio/low_mean": 0.0008322886860696599, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0050640220870263875, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 754.4500122070312, "completions/mean_terminated_length": 574.75, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.16197333578020334, "epoch": 0.08703071672354949, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022450095042586327, "kl": 0.010798165458254516, "learning_rate": 8.22184300341297e-06, "loss": -0.0295, "num_tokens": 16602399.0, "reward": 1.1750000715255737, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.8379271030426025, "sampling/importance_sampling_ratio/mean": 0.46785277128219604, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7541794776916504, "sampling/sampling_logp_difference/mean": 0.015796788036823273, "step": 153, "step_time": 71.78943335241638 }, { "clip_ratio/high_max": 0.009260544669814408, "clip_ratio/high_mean": 0.0035422442306298763, "clip_ratio/low_mean": 0.0024724882023292594, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006014732411131263, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 769.0, "completions/mean_length": 821.4750366210938, "completions/mean_terminated_length": 547.4705810546875, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.15912003628909588, "epoch": 0.0875995449374289, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02105317823588848, "kl": 0.0057942940620705485, "learning_rate": 8.216723549488056e-06, "loss": -0.0313, "num_tokens": 16707402.0, "reward": 0.75, "reward_std": 0.9268086552619934, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.789973735809326, "sampling/importance_sampling_ratio/mean": 0.4366072118282318, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.881043791770935, "sampling/sampling_logp_difference/mean": 0.01430858951061964, "step": 154, "step_time": 71.82289536809549 }, { "clip_ratio/high_max": 0.007971336890477687, "clip_ratio/high_mean": 0.0019881335392710753, "clip_ratio/low_mean": 0.0020263672195142135, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004014500795165077, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 744.375, "completions/mean_terminated_length": 576.5999755859375, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.13044185191392899, "epoch": 0.08816837315130831, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.026606110855937004, "kl": 0.004747559200040996, "learning_rate": 8.21160409556314e-06, "loss": -0.0243, "num_tokens": 16814889.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.6686720848083496, "sampling/importance_sampling_ratio/mean": 0.4829840660095215, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4192228317260742, "sampling/sampling_logp_difference/mean": 0.012953052297234535, "step": 155, "step_time": 72.5901672791224 }, { "clip_ratio/high_max": 0.01255795091856271, "clip_ratio/high_mean": 0.003901578937075101, "clip_ratio/low_mean": 0.0019287108952994458, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005830289941513911, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 880.0, "completions/mean_length": 900.6000366210938, "completions/mean_terminated_length": 644.3077392578125, "completions/min_length": 465.0, "completions/min_terminated_length": 465.0, "entropy": 0.15509542636573315, "epoch": 0.08873720136518772, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.048133112490177155, "kl": 0.0052733612537849694, "learning_rate": 8.206484641638225e-06, "loss": -0.1155, "num_tokens": 16927313.0, "reward": 0.6000000238418579, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.3928608894348145, "sampling/importance_sampling_ratio/mean": 0.4650731086730957, "sampling/importance_sampling_ratio/min": 0.0012556955916807055, "sampling/sampling_logp_difference/max": 2.071348190307617, "sampling/sampling_logp_difference/mean": 0.01446537859737873, "step": 156, "step_time": 75.55353289982304 }, { "clip_ratio/high_max": 0.012383618101011962, "clip_ratio/high_mean": 0.003904260534909554, "clip_ratio/low_mean": 0.0018528714535932522, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005757132166763768, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 872.5250244140625, "completions/mean_terminated_length": 620.0667114257812, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.17297805845737457, "epoch": 0.08930602957906712, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01876014471054077, "kl": 0.00476448837434873, "learning_rate": 8.201365187713311e-06, "loss": 0.0912, "num_tokens": 17035886.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.661101818084717, "sampling/importance_sampling_ratio/mean": 0.41720008850097656, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.104671001434326, "sampling/sampling_logp_difference/mean": 0.01453197281807661, "step": 157, "step_time": 73.79758433508687 }, { "clip_ratio/high_max": 0.014485916239209473, "clip_ratio/high_mean": 0.004974574520019814, "clip_ratio/low_mean": 0.003198242178768851, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.008172816422302276, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 888.5250244140625, "completions/mean_terminated_length": 662.7333374023438, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.11550551373511553, "epoch": 0.08987485779294653, "frac_reward_zero_std": 0.0, "grad_norm": 0.05467712879180908, "kl": 0.0072507365548517555, "learning_rate": 8.196245733788397e-06, "loss": -0.0268, "num_tokens": 17149739.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.4762794971466064, "sampling/importance_sampling_ratio/mean": 0.4830264449119568, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.333915948867798, "sampling/sampling_logp_difference/mean": 0.012375887483358383, "step": 158, "step_time": 75.97083419584669 }, { "clip_ratio/high_max": 0.007068061386235058, "clip_ratio/high_mean": 0.0014387126720976084, "clip_ratio/low_mean": 0.0010253906184516381, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00246410327963531, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 849.4000244140625, "completions/mean_terminated_length": 486.7692565917969, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.06818535085767508, "epoch": 0.09044368600682594, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024312889203429222, "kl": 0.01122452865820378, "learning_rate": 8.191126279863482e-06, "loss": -0.0058, "num_tokens": 17249491.0, "reward": 0.6000000238418579, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.163792848587036, "sampling/importance_sampling_ratio/mean": 0.46965599060058594, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4547972679138184, "sampling/sampling_logp_difference/mean": 0.009284456260502338, "step": 159, "step_time": 69.3568821856752 }, { "clip_ratio/high_max": 0.004901623236946762, "clip_ratio/high_mean": 0.0014568713231710717, "clip_ratio/low_mean": 0.0024414062063442543, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038982774422038347, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 988.7250366210938, "completions/mean_terminated_length": 822.4285888671875, "completions/min_length": 630.0, "completions/min_terminated_length": 630.0, "entropy": 0.14005356281995773, "epoch": 0.09101251422070535, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014966469258069992, "kl": 0.0037609988648910075, "learning_rate": 8.186006825938566e-06, "loss": -0.0385, "num_tokens": 17358784.0, "reward": 0.25, "reward_std": 0.588348388671875, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 1.1684995889663696, "sampling/importance_sampling_ratio/mean": 0.36701858043670654, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.355405569076538, "sampling/sampling_logp_difference/mean": 0.012344762682914734, "step": 160, "step_time": 75.97768434439786 }, { "clip_ratio/high_max": 0.0036086569307371974, "clip_ratio/high_mean": 0.0011285459040664136, "clip_ratio/low_mean": 0.0017089843604480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028375302790664136, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 981.9750366210938, "completions/mean_terminated_length": 687.7999877929688, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.11658922489732504, "epoch": 0.09158134243458475, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02181297168135643, "kl": 0.0035625831224024296, "learning_rate": 8.180887372013653e-06, "loss": 0.0188, "num_tokens": 17469711.0, "reward": 0.20000000298023224, "reward_std": 0.5638694167137146, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.585117816925049, "sampling/importance_sampling_ratio/mean": 0.5666013956069946, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.194735050201416, "sampling/sampling_logp_difference/mean": 0.011165943928062916, "step": 161, "step_time": 73.36597706074826 }, { "clip_ratio/high_max": 0.009847222885582596, "clip_ratio/high_mean": 0.0029329896351555362, "clip_ratio/low_mean": 0.003124296577880159, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.006057286111172289, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 843.75, "completions/mean_terminated_length": 644.5263061523438, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "entropy": 0.11765863001346588, "epoch": 0.09215017064846416, "frac_reward_zero_std": 0.0, "grad_norm": 0.02308034338057041, "kl": 0.004880258813500404, "learning_rate": 8.175767918088737e-06, "loss": -0.0162, "num_tokens": 17563117.0, "reward": 0.699999988079071, "reward_std": 0.822753369808197, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 1.6686639785766602, "sampling/importance_sampling_ratio/mean": 0.34538474678993225, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9817715883255005, "sampling/sampling_logp_difference/mean": 0.012182227335870266, "step": 162, "step_time": 72.48426746577024 }, { "clip_ratio/high_max": 0.011308298679068685, "clip_ratio/high_mean": 0.003557496253051795, "clip_ratio/low_mean": 0.000903320336874458, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004460816620849073, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 624.0, "completions/mean_length": 848.4000244140625, "completions/mean_terminated_length": 483.69232177734375, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.07395020080730319, "epoch": 0.09271899886234357, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01499019656330347, "kl": 0.004304291418520734, "learning_rate": 8.170648464163822e-06, "loss": 0.0039, "num_tokens": 17673669.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.971607208251953, "sampling/importance_sampling_ratio/mean": 0.43816128373146057, "sampling/importance_sampling_ratio/min": 0.009331800043582916, "sampling/sampling_logp_difference/max": 1.629730224609375, "sampling/sampling_logp_difference/mean": 0.009619594551622868, "step": 163, "step_time": 75.96528866328299 }, { "clip_ratio/high_max": 0.013111047970596701, "clip_ratio/high_mean": 0.004885599060798995, "clip_ratio/low_mean": 0.0011962890821450856, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006081888132030144, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 843.0, "completions/mean_length": 848.6500244140625, "completions/mean_terminated_length": 611.4117431640625, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.11300551984459162, "epoch": 0.09328782707622298, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015677446499466896, "kl": 0.01282639493001625, "learning_rate": 8.165529010238908e-06, "loss": 0.0652, "num_tokens": 17777239.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.8382948637008667, "sampling/importance_sampling_ratio/mean": 0.364892840385437, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.478618621826172, "sampling/sampling_logp_difference/mean": 0.011743452399969101, "step": 164, "step_time": 72.64049674128182 }, { "clip_ratio/high_max": 0.004771651001647115, "clip_ratio/high_mean": 0.0009543301857775077, "clip_ratio/low_mean": 0.0018554687121650204, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002809798883390613, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 831.8250122070312, "completions/mean_terminated_length": 639.6500244140625, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.12105104606598616, "epoch": 0.09385665529010238, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02902846969664097, "kl": 0.006999443925451487, "learning_rate": 8.160409556313993e-06, "loss": 0.0061, "num_tokens": 17878888.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.0618836879730225, "sampling/importance_sampling_ratio/mean": 0.42580968141555786, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4637155532836914, "sampling/sampling_logp_difference/mean": 0.013119480572640896, "step": 165, "step_time": 75.94690015004016 }, { "clip_ratio/high_max": 0.012543157674372196, "clip_ratio/high_mean": 0.004174619447439909, "clip_ratio/low_mean": 0.0015049540779727977, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005679573456291109, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 889.0, "completions/mean_length": 881.125, "completions/mean_terminated_length": 666.8125, "completions/min_length": 499.0, "completions/min_terminated_length": 499.0, "entropy": 0.16528316028416157, "epoch": 0.09442548350398179, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024498876184225082, "kl": 0.007618301548063755, "learning_rate": 8.155290102389079e-06, "loss": 0.0428, "num_tokens": 17990021.0, "reward": 0.6000000238418579, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.6495490074157715, "sampling/importance_sampling_ratio/mean": 0.5514043569564819, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.179382801055908, "sampling/sampling_logp_difference/mean": 0.014545339159667492, "step": 166, "step_time": 75.73297706316225 }, { "clip_ratio/high_max": 0.01440528966486454, "clip_ratio/high_mean": 0.0031185876578092575, "clip_ratio/low_mean": 0.0012939453044964466, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004412532958667725, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 922.0, "completions/mean_length": 971.4750366210938, "completions/mean_terminated_length": 761.375, "completions/min_length": 471.0, "completions/min_terminated_length": 471.0, "entropy": 0.12320612650364637, "epoch": 0.0949943117178612, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03128295764327049, "kl": 0.004022381181130186, "learning_rate": 8.150170648464164e-06, "loss": 0.1356, "num_tokens": 18108088.0, "reward": 0.375, "reward_std": 0.7741827964782715, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.9812965393066406, "sampling/importance_sampling_ratio/mean": 0.5962942242622375, "sampling/importance_sampling_ratio/min": 0.014258257113397121, "sampling/sampling_logp_difference/max": 2.2502126693725586, "sampling/sampling_logp_difference/mean": 0.011976862326264381, "step": 167, "step_time": 77.38058993802406 }, { "clip_ratio/high_max": 0.012169579393230379, "clip_ratio/high_mean": 0.002878836734453216, "clip_ratio/low_mean": 0.0015136718939174898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004392508650198579, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 953.375, "completions/mean_terminated_length": 741.5, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.11572164390236139, "epoch": 0.09556313993174062, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.012106397189199924, "kl": 0.004163578705629334, "learning_rate": 8.14505119453925e-06, "loss": 0.0323, "num_tokens": 18213055.0, "reward": 0.4749999940395355, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.6061158180236816, "sampling/importance_sampling_ratio/mean": 0.5139811635017395, "sampling/importance_sampling_ratio/min": 0.012110701762139797, "sampling/sampling_logp_difference/max": 1.5233652591705322, "sampling/sampling_logp_difference/mean": 0.01045342069119215, "step": 168, "step_time": 74.50717924698256 }, { "clip_ratio/high_max": 0.013112637447193265, "clip_ratio/high_mean": 0.003428229159908369, "clip_ratio/low_mean": 0.0018406859271635767, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005268915090709925, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 889.375, "completions/mean_terminated_length": 609.7692260742188, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.10920702945441008, "epoch": 0.09613196814562003, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014758195728063583, "kl": 0.005793802090920508, "learning_rate": 8.139931740614334e-06, "loss": -0.0044, "num_tokens": 18320918.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.5239993333816528, "sampling/importance_sampling_ratio/mean": 0.3616368770599365, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.016087532043457, "sampling/sampling_logp_difference/mean": 0.011213175021111965, "step": 169, "step_time": 74.38601183053106 }, { "clip_ratio/high_max": 0.013665110222063959, "clip_ratio/high_mean": 0.00446597289192141, "clip_ratio/low_mean": 0.001489257821958745, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005955230561085045, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 847.9000244140625, "completions/mean_terminated_length": 583.75, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.1278279647231102, "epoch": 0.09670079635949944, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03647501766681671, "kl": 0.007778439612593502, "learning_rate": 8.13481228668942e-06, "loss": -0.0203, "num_tokens": 18427258.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.847644567489624, "sampling/importance_sampling_ratio/mean": 0.45988979935646057, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9622588157653809, "sampling/sampling_logp_difference/mean": 0.013042653910815716, "step": 170, "step_time": 77.41984597989358 }, { "clip_ratio/high_max": 0.010504274221602827, "clip_ratio/high_mean": 0.0028778293635696173, "clip_ratio/low_mean": 0.002267015486722812, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005144844762980938, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 873.2250366210938, "completions/mean_terminated_length": 593.2142944335938, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.14227797649800777, "epoch": 0.09726962457337884, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01683744229376316, "kl": 0.0048455205105710775, "learning_rate": 8.129692832764505e-06, "loss": 0.0843, "num_tokens": 18516035.0, "reward": 0.625, "reward_std": 0.8969321250915527, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.5258100032806396, "sampling/importance_sampling_ratio/mean": 0.4303147792816162, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2617578506469727, "sampling/sampling_logp_difference/mean": 0.013728179037570953, "step": 171, "step_time": 71.02309834654443 }, { "clip_ratio/high_max": 0.008232058462454006, "clip_ratio/high_mean": 0.0018003525474341586, "clip_ratio/low_mean": 0.0006238675923668779, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002424220132525079, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 812.0, "completions/mean_length": 822.7000122070312, "completions/mean_terminated_length": 576.6666870117188, "completions/min_length": 387.0, "completions/min_terminated_length": 387.0, "entropy": 0.10782715678215027, "epoch": 0.09783845278725825, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.030807092785835266, "kl": 0.010095924837514758, "learning_rate": 8.12457337883959e-06, "loss": 0.1177, "num_tokens": 18621935.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.8394737243652344, "sampling/importance_sampling_ratio/mean": 0.6093837022781372, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5988032817840576, "sampling/sampling_logp_difference/mean": 0.011823609471321106, "step": 172, "step_time": 72.45441179396585 }, { "clip_ratio/high_max": 0.015098561067134142, "clip_ratio/high_mean": 0.00444334142957814, "clip_ratio/low_mean": 0.0014160156570142135, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0058593571884557605, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 847.7000122070312, "completions/mean_terminated_length": 583.25, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.0951362568885088, "epoch": 0.09840728100113766, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015831351280212402, "kl": 0.009153847116976976, "learning_rate": 8.119453924914676e-06, "loss": 0.0094, "num_tokens": 18727971.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.1409850120544434, "sampling/importance_sampling_ratio/mean": 0.5416771173477173, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.891223430633545, "sampling/sampling_logp_difference/mean": 0.010806932114064693, "step": 173, "step_time": 74.46937549184076 }, { "clip_ratio/high_max": 0.011367419734597206, "clip_ratio/high_mean": 0.0036286030226619914, "clip_ratio/low_mean": 0.0005126953037688509, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004141298326430842, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 1024.0, "completions/max_terminated_length": 968.0, "completions/mean_length": 633.2750244140625, "completions/mean_terminated_length": 564.3235473632812, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.16082824394106865, "epoch": 0.09897610921501707, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.030928419902920723, "kl": 0.01391127350507304, "learning_rate": 8.114334470989763e-06, "loss": -0.0427, "num_tokens": 18816518.0, "reward": 1.3000000715255737, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.8500000238418579, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.258582353591919, "sampling/importance_sampling_ratio/mean": 0.5471078157424927, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7111072540283203, "sampling/sampling_logp_difference/mean": 0.017037099227309227, "step": 174, "step_time": 69.89194372179918 }, { "clip_ratio/high_max": 0.009237889433279634, "clip_ratio/high_mean": 0.0024527035420760512, "clip_ratio/low_mean": 0.0008789062412688509, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033316097542410716, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 653.0, "completions/mean_length": 794.2000122070312, "completions/mean_terminated_length": 449.5, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.11597143393009901, "epoch": 0.09954493742889647, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.035118669271469116, "kl": 0.011961875658016652, "learning_rate": 8.109215017064847e-06, "loss": -0.0654, "num_tokens": 18922478.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.675896406173706, "sampling/importance_sampling_ratio/mean": 0.3843359649181366, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.635894775390625, "sampling/sampling_logp_difference/mean": 0.012332558631896973, "step": 175, "step_time": 75.29679386503994 }, { "clip_ratio/high_max": 0.015709952567704022, "clip_ratio/high_mean": 0.004376774712000042, "clip_ratio/low_mean": 0.0024658202892169356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006842595059424639, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 888.875, "completions/mean_terminated_length": 637.9285888671875, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.14191098418086767, "epoch": 0.10011376564277588, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022740265354514122, "kl": 0.01824980793753639, "learning_rate": 8.104095563139932e-06, "loss": 0.111, "num_tokens": 19031409.0, "reward": 0.625, "reward_std": 0.8969321250915527, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.7855112552642822, "sampling/importance_sampling_ratio/mean": 0.4109903872013092, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7038094997406006, "sampling/sampling_logp_difference/mean": 0.013513374142348766, "step": 176, "step_time": 76.41022808919661 }, { "clip_ratio/high_max": 0.00976282509509474, "clip_ratio/high_mean": 0.0021264176320983097, "clip_ratio/low_mean": 0.001629630962270312, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037560485652647913, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 907.5, "completions/mean_terminated_length": 506.22222900390625, "completions/min_length": 427.0, "completions/min_terminated_length": 427.0, "entropy": 0.14678637497127056, "epoch": 0.10068259385665529, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.010407417081296444, "kl": 0.005654433916788548, "learning_rate": 8.098976109215018e-06, "loss": 0.0197, "num_tokens": 19138357.0, "reward": 0.4000000059604645, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.0733838081359863, "sampling/importance_sampling_ratio/mean": 0.501867413520813, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6570227146148682, "sampling/sampling_logp_difference/mean": 0.014465207234025002, "step": 177, "step_time": 71.66929330374114 }, { "clip_ratio/high_max": 0.006057700142264366, "clip_ratio/high_mean": 0.0012115400459151715, "clip_ratio/low_mean": 0.0010742187732830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022857587464386597, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 973.4750366210938, "completions/mean_terminated_length": 619.7999877929688, "completions/min_length": 446.0, "completions/min_terminated_length": 446.0, "entropy": 0.15673729870468378, "epoch": 0.1012514220705347, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.018501803278923035, "kl": 0.004527091106865555, "learning_rate": 8.093856655290103e-06, "loss": 0.1257, "num_tokens": 19245440.0, "reward": 0.15000000596046448, "reward_std": 0.48304587602615356, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.05000000074505806, "rewards/gpt_judge_reward/std": 0.22072142362594604, "sampling/importance_sampling_ratio/max": 2.3066916465759277, "sampling/importance_sampling_ratio/mean": 0.5651032328605652, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7707273960113525, "sampling/sampling_logp_difference/mean": 0.013024384155869484, "step": 178, "step_time": 76.69053986901417 }, { "clip_ratio/high_max": 0.010300195368472487, "clip_ratio/high_mean": 0.0024417793611064553, "clip_ratio/low_mean": 0.0021728515639551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00461463097599335, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 978.0, "completions/mean_length": 846.4500122070312, "completions/mean_terminated_length": 650.2105102539062, "completions/min_length": 371.0, "completions/min_terminated_length": 371.0, "entropy": 0.13162186928093433, "epoch": 0.1018202502844141, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.061773162335157394, "kl": 0.008450915571302176, "learning_rate": 8.088737201365187e-06, "loss": -0.1847, "num_tokens": 19351042.0, "reward": 0.9000000357627869, "reward_std": 0.9818872809410095, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.9563658237457275, "sampling/importance_sampling_ratio/mean": 0.5369168519973755, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.118889808654785, "sampling/sampling_logp_difference/mean": 0.013674936257302761, "step": 179, "step_time": 73.08169463183731 }, { "clip_ratio/high_max": 0.008120390004478395, "clip_ratio/high_mean": 0.0017926173459272832, "clip_ratio/low_mean": 0.0016601562674622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034527734387665987, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 942.0, "completions/mean_length": 963.0250244140625, "completions/mean_terminated_length": 617.5, "completions/min_length": 392.0, "completions/min_terminated_length": 392.0, "entropy": 0.11318887211382389, "epoch": 0.10238907849829351, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02434244379401207, "kl": 0.004647872352506965, "learning_rate": 8.083617747440273e-06, "loss": 0.0525, "num_tokens": 19459507.0, "reward": 0.30000001192092896, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.6409831047058105, "sampling/importance_sampling_ratio/mean": 0.5986484289169312, "sampling/importance_sampling_ratio/min": 0.008867175318300724, "sampling/sampling_logp_difference/max": 3.1278159618377686, "sampling/sampling_logp_difference/mean": 0.011357361450791359, "step": 180, "step_time": 75.63208945421502 }, { "clip_ratio/high_max": 0.014047777047380805, "clip_ratio/high_mean": 0.006376532604917884, "clip_ratio/low_mean": 0.0009033203314174898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007279852929059416, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 657.7000122070312, "completions/mean_terminated_length": 551.3547973632812, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.15343792084604502, "epoch": 0.10295790671217292, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.040096912533044815, "kl": 0.014389359625056386, "learning_rate": 8.078498293515358e-06, "loss": 0.0522, "num_tokens": 19561895.0, "reward": 1.399999976158142, "reward_std": 0.8412444591522217, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.6363213062286377, "sampling/importance_sampling_ratio/mean": 0.5296415686607361, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3128790855407715, "sampling/sampling_logp_difference/mean": 0.015782583504915237, "step": 181, "step_time": 72.39985033217818 }, { "clip_ratio/high_max": 0.012094518810044974, "clip_ratio/high_mean": 0.003139862645184621, "clip_ratio/low_mean": 0.0015683339479437564, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00470819664769806, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 807.5, "completions/mean_terminated_length": 446.66668701171875, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.07028352515771985, "epoch": 0.10352673492605233, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02910131774842739, "kl": 0.006505472323624417, "learning_rate": 8.073378839590443e-06, "loss": 0.0077, "num_tokens": 19664203.0, "reward": 0.625, "reward_std": 0.8678739070892334, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.553194522857666, "sampling/importance_sampling_ratio/mean": 0.470651239156723, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1919033527374268, "sampling/sampling_logp_difference/mean": 0.009717203676700592, "step": 182, "step_time": 70.40071046282537 }, { "clip_ratio/high_max": 0.0037266092840582132, "clip_ratio/high_mean": 0.0007453218568116426, "clip_ratio/low_mean": 0.001464843759094947, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002210165577707812, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 900.0, "completions/mean_length": 991.9750366210938, "completions/mean_terminated_length": 703.75, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "entropy": 0.14142644312232733, "epoch": 0.10409556313993173, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.010906873270869255, "kl": 0.005271085363347083, "learning_rate": 8.068259385665529e-06, "loss": -0.0178, "num_tokens": 19774706.0, "reward": 0.17499999701976776, "reward_std": 0.5494753122329712, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 1.56693434715271, "sampling/importance_sampling_ratio/mean": 0.2952384948730469, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7861111164093018, "sampling/sampling_logp_difference/mean": 0.012459036894142628, "step": 183, "step_time": 74.82935679075308 }, { "clip_ratio/high_max": 0.00620361631445121, "clip_ratio/high_mean": 0.0017776637214410584, "clip_ratio/low_mean": 0.0012939453263243195, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003071609025937505, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 893.0, "completions/mean_length": 946.3250122070312, "completions/mean_terminated_length": 678.7777709960938, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.08931153360754251, "epoch": 0.10466439135381114, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.026185285300016403, "kl": 0.005225952045293525, "learning_rate": 8.063139931740615e-06, "loss": 0.0911, "num_tokens": 19880703.0, "reward": 0.45000001788139343, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.9890477657318115, "sampling/importance_sampling_ratio/mean": 0.6239909529685974, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5506961345672607, "sampling/sampling_logp_difference/mean": 0.009882968850433826, "step": 184, "step_time": 72.75763651588932 }, { "clip_ratio/high_max": 0.013034346629865468, "clip_ratio/high_mean": 0.005117734115628991, "clip_ratio/low_mean": 0.0017160759925900493, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.006833809980889782, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 752.4750366210938, "completions/mean_terminated_length": 506.8095397949219, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.08460564911365509, "epoch": 0.10523321956769056, "frac_reward_zero_std": 0.0, "grad_norm": 0.038176264613866806, "kl": 0.00806005927734077, "learning_rate": 8.0580204778157e-06, "loss": 0.0355, "num_tokens": 19979626.0, "reward": 1.0, "reward_std": 0.9870962500572205, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.248095989227295, "sampling/importance_sampling_ratio/mean": 0.44998449087142944, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9860259294509888, "sampling/sampling_logp_difference/mean": 0.011118188500404358, "step": 185, "step_time": 71.89646129915491 }, { "clip_ratio/high_max": 0.008519576687831432, "clip_ratio/high_mean": 0.0022489837574539706, "clip_ratio/low_mean": 0.002807617129292339, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0050566008721943945, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 939.5499877929688, "completions/mean_terminated_length": 648.6666870117188, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.15471921022981405, "epoch": 0.10580204778156997, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027302179485559464, "kl": 0.014450488961301744, "learning_rate": 8.052901023890784e-06, "loss": -0.0168, "num_tokens": 20091096.0, "reward": 0.42500001192092896, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290207743644714, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509575605392456, "sampling/importance_sampling_ratio/max": 1.6991037130355835, "sampling/importance_sampling_ratio/mean": 0.3617474138736725, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.080824613571167, "sampling/sampling_logp_difference/mean": 0.014588278718292713, "step": 186, "step_time": 74.48461768473499 }, { "clip_ratio/high_max": 0.006913899851497263, "clip_ratio/high_mean": 0.002164928133424837, "clip_ratio/low_mean": 0.0029221409295132617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00508706911932677, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 875.0, "completions/mean_length": 874.1749877929688, "completions/mean_terminated_length": 595.9285888671875, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "entropy": 0.10534181445837021, "epoch": 0.10637087599544938, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022924993187189102, "kl": 0.006812104780692607, "learning_rate": 8.04778156996587e-06, "loss": 0.0924, "num_tokens": 20187215.0, "reward": 0.550000011920929, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.1713180541992188, "sampling/importance_sampling_ratio/mean": 0.4512898325920105, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4553465843200684, "sampling/sampling_logp_difference/mean": 0.011258344165980816, "step": 187, "step_time": 74.15310828434303 }, { "clip_ratio/high_max": 0.010723607032559812, "clip_ratio/high_mean": 0.002919286285759881, "clip_ratio/low_mean": 0.001855468755820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004774754954269156, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 917.9249877929688, "completions/mean_terminated_length": 599.7000122070312, "completions/min_length": 396.0, "completions/min_terminated_length": 396.0, "entropy": 0.1532398033887148, "epoch": 0.10693970420932879, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029927557334303856, "kl": 0.006262617593165487, "learning_rate": 8.042662116040955e-06, "loss": 0.1366, "num_tokens": 20292556.0, "reward": 0.45000001788139343, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.5505764484405518, "sampling/importance_sampling_ratio/mean": 0.5305172801017761, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.118644952774048, "sampling/sampling_logp_difference/mean": 0.01298669446259737, "step": 188, "step_time": 74.85316241998225 }, { "clip_ratio/high_max": 0.015454372158274055, "clip_ratio/high_mean": 0.005913075961871073, "clip_ratio/low_mean": 0.0007465267917723395, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006659602659055963, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 650.9000244140625, "completions/mean_terminated_length": 471.25927734375, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.14842420164495707, "epoch": 0.1075085324232082, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0184069462120533, "kl": 0.013294170144945383, "learning_rate": 8.037542662116041e-06, "loss": -0.0541, "num_tokens": 20398224.0, "reward": 1.3000000715255737, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.044102430343628, "sampling/importance_sampling_ratio/mean": 0.34863027930259705, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5923056602478027, "sampling/sampling_logp_difference/mean": 0.016234099864959717, "step": 189, "step_time": 72.14289655722678 }, { "clip_ratio/high_max": 0.011752614242141135, "clip_ratio/high_mean": 0.0033504955390526447, "clip_ratio/low_mean": 0.0016113281490106601, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0049618236953392625, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 928.7250366210938, "completions/mean_terminated_length": 677.5454711914062, "completions/min_length": 437.0, "completions/min_terminated_length": 437.0, "entropy": 0.11845656391233206, "epoch": 0.1080773606370876, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03129059448838234, "kl": 0.0051862833788618445, "learning_rate": 8.032423208191126e-06, "loss": -0.0063, "num_tokens": 20510125.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.7209079265594482, "sampling/importance_sampling_ratio/mean": 0.4168340265750885, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8851255178451538, "sampling/sampling_logp_difference/mean": 0.012124625965952873, "step": 190, "step_time": 79.06717953481711 }, { "clip_ratio/high_max": 0.010789188789203763, "clip_ratio/high_mean": 0.003328186518047005, "clip_ratio/low_mean": 0.0031982422224245965, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.006526428507640958, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 970.0, "completions/mean_length": 919.6000366210938, "completions/mean_terminated_length": 676.0, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.12432592734694481, "epoch": 0.10864618885096701, "frac_reward_zero_std": 0.0, "grad_norm": 0.021168647333979607, "kl": 0.006104574247729033, "learning_rate": 8.027303754266212e-06, "loss": -0.0082, "num_tokens": 20619573.0, "reward": 0.5250000357627869, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 1.638893723487854, "sampling/importance_sampling_ratio/mean": 0.35585227608680725, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.504913806915283, "sampling/sampling_logp_difference/mean": 0.012323524802923203, "step": 191, "step_time": 74.39535854198039 }, { "clip_ratio/high_max": 0.0144479168811813, "clip_ratio/high_mean": 0.0050107916904380545, "clip_ratio/low_mean": 0.0032675803668098524, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008278371882624924, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 931.0, "completions/mean_length": 842.375, "completions/mean_terminated_length": 660.75, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "entropy": 0.13601310830563307, "epoch": 0.10921501706484642, "frac_reward_zero_std": 0.0, "grad_norm": 0.026274241507053375, "kl": 0.007605973398312926, "learning_rate": 8.022184300341297e-06, "loss": 0.0037, "num_tokens": 20727340.0, "reward": 0.699999988079071, "reward_std": 0.7909747362136841, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.4730567932128906, "sampling/importance_sampling_ratio/mean": 0.4044792652130127, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.836533546447754, "sampling/sampling_logp_difference/mean": 0.01425944734364748, "step": 192, "step_time": 73.93184130173177 }, { "clip_ratio/high_max": 0.011049014574382454, "clip_ratio/high_mean": 0.0031258398230420426, "clip_ratio/low_mean": 0.002315538178663701, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005441377870738506, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 938.0, "completions/mean_length": 909.0250244140625, "completions/mean_terminated_length": 564.1000366210938, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.1581739243119955, "epoch": 0.10978384527872583, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04098157212138176, "kl": 0.007251803879626095, "learning_rate": 8.017064846416383e-06, "loss": -0.1288, "num_tokens": 20829901.0, "reward": 0.4749999940395355, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.7915220260620117, "sampling/importance_sampling_ratio/mean": 0.5440300703048706, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.043750762939453, "sampling/sampling_logp_difference/mean": 0.014018476009368896, "step": 193, "step_time": 72.04165391251445 }, { "clip_ratio/high_max": 0.008226866193581372, "clip_ratio/high_mean": 0.0030137687863316387, "clip_ratio/low_mean": 0.001879882809589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00489365158136934, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 871.9000244140625, "completions/mean_terminated_length": 703.7894897460938, "completions/min_length": 452.0, "completions/min_terminated_length": 452.0, "entropy": 0.19245954975485802, "epoch": 0.11035267349260523, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.028199028223752975, "kl": 0.0060822973027825356, "learning_rate": 8.011945392491468e-06, "loss": 0.0816, "num_tokens": 20928961.0, "reward": 0.925000011920929, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.302490472793579, "sampling/importance_sampling_ratio/mean": 0.4997522234916687, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.137237548828125, "sampling/sampling_logp_difference/mean": 0.017151448875665665, "step": 194, "step_time": 73.54809640394524 }, { "clip_ratio/high_max": 0.011528661358170211, "clip_ratio/high_mean": 0.0038787656085332856, "clip_ratio/low_mean": 0.00164859574942966, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005527361267013475, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 905.0, "completions/mean_length": 814.0750122070312, "completions/mean_terminated_length": 604.1500244140625, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.1567178051918745, "epoch": 0.11092150170648464, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019064392894506454, "kl": 0.009222823078744113, "learning_rate": 8.006825938566552e-06, "loss": -0.0006, "num_tokens": 21025156.0, "reward": 0.824999988079071, "reward_std": 0.9026315212249756, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.590202808380127, "sampling/importance_sampling_ratio/mean": 0.3376808762550354, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9937841892242432, "sampling/sampling_logp_difference/mean": 0.01567893661558628, "step": 195, "step_time": 71.88640681817196 }, { "clip_ratio/high_max": 0.00718012178549543, "clip_ratio/high_mean": 0.0016773296083556488, "clip_ratio/low_mean": 0.0014160156351863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030933453017496504, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 942.0, "completions/mean_length": 911.7000122070312, "completions/mean_terminated_length": 574.7999877929688, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.09564378578215837, "epoch": 0.11149032992036405, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.031407181173563004, "kl": 0.007103330979589373, "learning_rate": 8.001706484641639e-06, "loss": -0.003, "num_tokens": 21137432.0, "reward": 0.45000001788139343, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.7485718727111816, "sampling/importance_sampling_ratio/mean": 0.392846941947937, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1339271068573, "sampling/sampling_logp_difference/mean": 0.00989393051713705, "step": 196, "step_time": 77.85513613419607 }, { "clip_ratio/high_max": 0.009753132588230073, "clip_ratio/high_mean": 0.0025728462205734104, "clip_ratio/low_mean": 0.0023193359229480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004892182245384902, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.0, "completions/mean_length": 923.7750244140625, "completions/mean_terminated_length": 623.1000366210938, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.15667933877557516, "epoch": 0.11205915813424346, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023383699357509613, "kl": 0.008642012573545799, "learning_rate": 7.996587030716725e-06, "loss": -0.0323, "num_tokens": 21240607.0, "reward": 0.5, "reward_std": 0.8770580887794495, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.730532169342041, "sampling/importance_sampling_ratio/mean": 0.3787885904312134, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5658130645751953, "sampling/sampling_logp_difference/mean": 0.014262489043176174, "step": 197, "step_time": 74.68828618479893 }, { "clip_ratio/high_max": 0.010842946008779109, "clip_ratio/high_mean": 0.002565893722930923, "clip_ratio/low_mean": 0.0011718749883584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037377686967374757, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 839.125, "completions/mean_terminated_length": 561.8125, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.12888417299836874, "epoch": 0.11262798634812286, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022366290912032127, "kl": 0.008480876102112234, "learning_rate": 7.99146757679181e-06, "loss": -0.0301, "num_tokens": 21345852.0, "reward": 0.7250000238418579, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.190711736679077, "sampling/importance_sampling_ratio/mean": 0.3090168833732605, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.389617919921875, "sampling/sampling_logp_difference/mean": 0.013410632498562336, "step": 198, "step_time": 75.67696451814845 }, { "clip_ratio/high_max": 0.012226614344399422, "clip_ratio/high_mean": 0.003265711638960056, "clip_ratio/low_mean": 0.0032226562761934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006488367915153503, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 954.0, "completions/mean_length": 910.375, "completions/mean_terminated_length": 569.5, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.16903672367334366, "epoch": 0.11319681456200227, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02995036356151104, "kl": 0.007635656918864697, "learning_rate": 7.986348122866894e-06, "loss": 0.0273, "num_tokens": 21445603.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.2011497020721436, "sampling/importance_sampling_ratio/mean": 0.47736650705337524, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5071094036102295, "sampling/sampling_logp_difference/mean": 0.014628834091126919, "step": 199, "step_time": 72.9156943212729 }, { "clip_ratio/high_max": 0.009704936295747757, "clip_ratio/high_mean": 0.0025751255670911632, "clip_ratio/low_mean": 0.0008789062758296495, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034540318156359717, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 770.0, "completions/mean_length": 908.625, "completions/mean_terminated_length": 604.45458984375, "completions/min_length": 445.0, "completions/min_terminated_length": 445.0, "entropy": 0.13188413344323635, "epoch": 0.11376564277588168, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01762261614203453, "kl": 0.005886377242859453, "learning_rate": 7.98122866894198e-06, "loss": -0.0049, "num_tokens": 21556932.0, "reward": 0.375, "reward_std": 0.667467474937439, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.657606840133667, "sampling/importance_sampling_ratio/mean": 0.39186468720436096, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8198647499084473, "sampling/sampling_logp_difference/mean": 0.012895484454929829, "step": 200, "step_time": 75.00352590763941 }, { "clip_ratio/high_max": 0.012144963897299021, "clip_ratio/high_mean": 0.0037221227685222402, "clip_ratio/low_mean": 0.0026611327721184352, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006383255822584033, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 929.2999877929688, "completions/mean_terminated_length": 708.3333740234375, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.17309672851115465, "epoch": 0.11433447098976109, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03440103679895401, "kl": 0.006812784296926111, "learning_rate": 7.976109215017065e-06, "loss": -0.0444, "num_tokens": 21673656.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.172372579574585, "sampling/importance_sampling_ratio/mean": 0.3353291153907776, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.036741256713867, "sampling/sampling_logp_difference/mean": 0.015562188811600208, "step": 201, "step_time": 77.69256892101839 }, { "clip_ratio/high_max": 0.012563072727061808, "clip_ratio/high_mean": 0.00436018654727377, "clip_ratio/low_mean": 0.0020011747219541576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006361361243762076, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 807.0, "completions/mean_length": 796.2999877929688, "completions/mean_terminated_length": 454.75, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.10734163969755173, "epoch": 0.1149032992036405, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015876151621341705, "kl": 0.010870143945794553, "learning_rate": 7.97098976109215e-06, "loss": -0.0218, "num_tokens": 21785100.0, "reward": 0.75, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.8549354076385498, "sampling/importance_sampling_ratio/mean": 0.38171204924583435, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.942990779876709, "sampling/sampling_logp_difference/mean": 0.01236751489341259, "step": 202, "step_time": 74.26480330200866 }, { "clip_ratio/high_max": 0.012767785345204175, "clip_ratio/high_mean": 0.0043060576426796615, "clip_ratio/low_mean": 0.0020640745933633298, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006370132206939161, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 890.2999877929688, "completions/mean_terminated_length": 709.4117431640625, "completions/min_length": 476.0, "completions/min_terminated_length": 476.0, "entropy": 0.13307121861726046, "epoch": 0.11547212741751992, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030435560271143913, "kl": 0.0076968162902630866, "learning_rate": 7.965870307167236e-06, "loss": -0.0926, "num_tokens": 21891424.0, "reward": 0.75, "reward_std": 0.9268086552619934, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.0910775661468506, "sampling/importance_sampling_ratio/mean": 0.3865329921245575, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9640228748321533, "sampling/sampling_logp_difference/mean": 0.013600203208625317, "step": 203, "step_time": 74.34313221881166 }, { "clip_ratio/high_max": 0.0117752710939385, "clip_ratio/high_mean": 0.0049944166239583865, "clip_ratio/low_mean": 0.0018606682569952682, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006855084851849824, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 956.0, "completions/mean_length": 748.4000244140625, "completions/mean_terminated_length": 544.6956787109375, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.126746978610754, "epoch": 0.11604095563139932, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.041807081550359726, "kl": 0.029774026421364397, "learning_rate": 7.96075085324232e-06, "loss": 0.1729, "num_tokens": 21996280.0, "reward": 1.024999976158142, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.8622952699661255, "sampling/importance_sampling_ratio/mean": 0.2950308322906494, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.799450159072876, "sampling/sampling_logp_difference/mean": 0.014279372990131378, "step": 204, "step_time": 74.39821347012185 }, { "clip_ratio/high_max": 0.00842980423476547, "clip_ratio/high_mean": 0.0022682258568238467, "clip_ratio/low_mean": 0.0008300781264551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030983039614511654, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 965.9249877929688, "completions/mean_terminated_length": 692.1428833007812, "completions/min_length": 424.0, "completions/min_terminated_length": 424.0, "entropy": 0.08200011728331447, "epoch": 0.11660978384527873, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.008642220869660378, "kl": 0.005257279990473762, "learning_rate": 7.955631399317407e-06, "loss": -0.0336, "num_tokens": 22111133.0, "reward": 0.25, "reward_std": 0.588348388671875, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.522106170654297, "sampling/importance_sampling_ratio/mean": 0.47551918029785156, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.118410110473633, "sampling/sampling_logp_difference/mean": 0.009182844310998917, "step": 205, "step_time": 76.57315141381696 }, { "clip_ratio/high_max": 0.004987393680494279, "clip_ratio/high_mean": 0.0011151257785968482, "clip_ratio/low_mean": 0.002587890616268851, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003703016438521445, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 973.7250366210938, "completions/mean_terminated_length": 621.7999877929688, "completions/min_length": 425.0, "completions/min_terminated_length": 425.0, "entropy": 0.11407606583088636, "epoch": 0.11717861205915814, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.033922646194696426, "kl": 0.004836713458644226, "learning_rate": 7.950511945392491e-06, "loss": -0.0019, "num_tokens": 22216226.0, "reward": 0.22500000894069672, "reward_std": 0.6196566820144653, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.9863693714141846, "sampling/importance_sampling_ratio/mean": 0.595641553401947, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4489264488220215, "sampling/sampling_logp_difference/mean": 0.010610847733914852, "step": 206, "step_time": 73.21767536294647 }, { "clip_ratio/high_max": 0.008832869410980493, "clip_ratio/high_mean": 0.0025483370409347117, "clip_ratio/low_mean": 0.0012695312598225428, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003817868360783905, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 831.3250122070312, "completions/mean_terminated_length": 595.8333129882812, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.12013427913188934, "epoch": 0.11774744027303755, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.010822143405675888, "kl": 0.0052584498189389706, "learning_rate": 7.945392491467578e-06, "loss": 0.0083, "num_tokens": 22328503.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.05812668800354, "sampling/importance_sampling_ratio/mean": 0.4033892750740051, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.529998779296875, "sampling/sampling_logp_difference/mean": 0.013155372813344002, "step": 207, "step_time": 73.92913885042071 }, { "clip_ratio/high_max": 0.018090869052684866, "clip_ratio/high_mean": 0.005847866981639527, "clip_ratio/low_mean": 0.0013992314925417304, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007247098459629342, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 967.0, "completions/mean_length": 708.6500244140625, "completions/mean_terminated_length": 519.4400024414062, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.11363341193646193, "epoch": 0.11831626848691695, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019233563914895058, "kl": 0.015011729090474546, "learning_rate": 7.940273037542662e-06, "loss": -0.0046, "num_tokens": 22416761.0, "reward": 1.024999976158142, "reward_std": 0.8911962509155273, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.227745771408081, "sampling/importance_sampling_ratio/mean": 0.4648776948451996, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.179257392883301, "sampling/sampling_logp_difference/mean": 0.012616115622222424, "step": 208, "step_time": 68.76738625927828 }, { "clip_ratio/high_max": 0.007243256084620953, "clip_ratio/high_mean": 0.0018471664370736107, "clip_ratio/low_mean": 0.0013671874912688509, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032143539283424616, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 905.7250366210938, "completions/mean_terminated_length": 708.6000366210938, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "entropy": 0.14392353408038616, "epoch": 0.11888509670079636, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01833682507276535, "kl": 0.006520924216601998, "learning_rate": 7.935153583617748e-06, "loss": -0.0562, "num_tokens": 22518126.0, "reward": 0.675000011920929, "reward_std": 0.9167249798774719, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.212489604949951, "sampling/importance_sampling_ratio/mean": 0.43610209226608276, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.746410369873047, "sampling/sampling_logp_difference/mean": 0.013870556838810444, "step": 209, "step_time": 71.67713423655368 }, { "clip_ratio/high_max": 0.01153415348380804, "clip_ratio/high_mean": 0.003906359372194856, "clip_ratio/low_mean": 0.0024926970727392472, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006399056233931333, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 893.4000244140625, "completions/mean_terminated_length": 675.7333984375, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.17306817509233952, "epoch": 0.11945392491467577, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03220558539032936, "kl": 0.0074389761430211365, "learning_rate": 7.930034129692833e-06, "loss": -0.0827, "num_tokens": 22624070.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.8637187480926514, "sampling/importance_sampling_ratio/mean": 0.4704393446445465, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.099001407623291, "sampling/sampling_logp_difference/mean": 0.016181230545043945, "step": 210, "step_time": 75.03108632285148 }, { "clip_ratio/high_max": 0.0098783690482378, "clip_ratio/high_mean": 0.001975673789274879, "clip_ratio/low_mean": 0.0003394605155335739, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0023151343048084527, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 925.1000366210938, "completions/mean_terminated_length": 529.5, "completions/min_length": 440.0, "completions/min_terminated_length": 440.0, "entropy": 0.08559564594179392, "epoch": 0.12002275312855518, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.0077168699353933334, "kl": 0.004228778096148744, "learning_rate": 7.924914675767918e-06, "loss": 0.0123, "num_tokens": 22721218.0, "reward": 0.3499999940395355, "reward_std": 0.7355444431304932, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509575605392456, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 1.9470272064208984, "sampling/importance_sampling_ratio/mean": 0.294650137424469, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9115982055664062, "sampling/sampling_logp_difference/mean": 0.009801342152059078, "step": 211, "step_time": 73.64418886601925 }, { "clip_ratio/high_max": 0.003182434185873717, "clip_ratio/high_mean": 0.0006364868313539773, "clip_ratio/low_mean": 0.0007080078212311491, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013444946525851265, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 799.4249877929688, "completions/mean_terminated_length": 551.2105102539062, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "entropy": 0.11343553476035595, "epoch": 0.12059158134243458, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.024829735979437828, "kl": 0.006575206178240478, "learning_rate": 7.919795221843004e-06, "loss": -0.016, "num_tokens": 22816363.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.0181524753570557, "sampling/importance_sampling_ratio/mean": 0.47264283895492554, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8741220235824585, "sampling/sampling_logp_difference/mean": 0.012846330180764198, "step": 212, "step_time": 69.0430370781105 }, { "clip_ratio/high_max": 0.011983550037257373, "clip_ratio/high_mean": 0.0030290439317468554, "clip_ratio/low_mean": 0.002221679693320766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005250723770586774, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 837.0, "completions/mean_length": 918.0250244140625, "completions/mean_terminated_length": 600.1000366210938, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "entropy": 0.12260660156607628, "epoch": 0.12116040955631399, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027172550559043884, "kl": 0.0058789050381165, "learning_rate": 7.914675767918089e-06, "loss": -0.0304, "num_tokens": 22923148.0, "reward": 0.45000001788139343, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.6606653928756714, "sampling/importance_sampling_ratio/mean": 0.4104480743408203, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7038114070892334, "sampling/sampling_logp_difference/mean": 0.012076886370778084, "step": 213, "step_time": 72.4524942082353 }, { "clip_ratio/high_max": 0.009423389099538326, "clip_ratio/high_mean": 0.0023168057232396677, "clip_ratio/low_mean": 0.0013671874858118827, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003683993185404688, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 853.0, "completions/mean_length": 952.7250366210938, "completions/mean_terminated_length": 667.625, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.11835277080535889, "epoch": 0.1217292377701934, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011792579665780067, "kl": 0.008630549011286348, "learning_rate": 7.909556313993175e-06, "loss": 0.0279, "num_tokens": 23032417.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.520049810409546, "sampling/importance_sampling_ratio/mean": 0.3059108853340149, "sampling/importance_sampling_ratio/min": 0.0012476906413212419, "sampling/sampling_logp_difference/max": 4.197785377502441, "sampling/sampling_logp_difference/mean": 0.011712274514138699, "step": 214, "step_time": 78.77038354612887 }, { "clip_ratio/high_max": 0.001351538929156959, "clip_ratio/high_mean": 0.0003234992836951278, "clip_ratio/low_mean": 0.0011230468953726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014465461717918515, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 947.3250122070312, "completions/mean_terminated_length": 745.1818237304688, "completions/min_length": 464.0, "completions/min_terminated_length": 464.0, "entropy": 0.1415241491049528, "epoch": 0.1222980659840728, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.034104686230421066, "kl": 0.014696750906296074, "learning_rate": 7.90443686006826e-06, "loss": -0.046, "num_tokens": 23144022.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 1.9945228099822998, "sampling/importance_sampling_ratio/mean": 0.35658901929855347, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.5938034057617188, "sampling/sampling_logp_difference/mean": 0.012903207913041115, "step": 215, "step_time": 76.28457454359159 }, { "clip_ratio/high_max": 0.017966683604754508, "clip_ratio/high_mean": 0.008288423938211054, "clip_ratio/low_mean": 0.002102989223203622, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010391412943135947, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 709.375, "completions/mean_terminated_length": 618.0322265625, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.18177830427885056, "epoch": 0.12286689419795221, "frac_reward_zero_std": 0.0, "grad_norm": 0.057805221527814865, "kl": 0.012191205110866576, "learning_rate": 7.899317406143346e-06, "loss": 0.0334, "num_tokens": 23246605.0, "reward": 1.4250000715255737, "reward_std": 0.8439072966575623, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.761630058288574, "sampling/importance_sampling_ratio/mean": 0.6501700282096863, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.351985454559326, "sampling/sampling_logp_difference/mean": 0.018218044191598892, "step": 216, "step_time": 73.4079705211334 }, { "clip_ratio/high_max": 0.014509266766253859, "clip_ratio/high_mean": 0.00493089749943465, "clip_ratio/low_mean": 0.0026611328503349796, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007592030277010053, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 881.3500366210938, "completions/mean_terminated_length": 707.0, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "entropy": 0.1474325992166996, "epoch": 0.12343572241183162, "frac_reward_zero_std": 0.0, "grad_norm": 0.0205471683293581, "kl": 0.007432444195728749, "learning_rate": 7.89419795221843e-06, "loss": 0.0818, "num_tokens": 23349443.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.19606614112854, "sampling/importance_sampling_ratio/mean": 0.5012333393096924, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2563352584838867, "sampling/sampling_logp_difference/mean": 0.014147952198982239, "step": 217, "step_time": 74.33520090463571 }, { "clip_ratio/high_max": 0.01122825377387926, "clip_ratio/high_mean": 0.0036277756589697674, "clip_ratio/low_mean": 0.0014160156133584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005043791403295472, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 739.1749877929688, "completions/mean_terminated_length": 506.1363830566406, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.15862775780260563, "epoch": 0.12400455062571103, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02365173026919365, "kl": 0.008607629861216992, "learning_rate": 7.889078498293515e-06, "loss": 0.0084, "num_tokens": 23447650.0, "reward": 0.9750000238418579, "reward_std": 0.9469925761222839, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.242762327194214, "sampling/importance_sampling_ratio/mean": 0.4198482036590576, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6046762466430664, "sampling/sampling_logp_difference/mean": 0.016105709597468376, "step": 218, "step_time": 69.46292073000222 }, { "clip_ratio/high_max": 0.01616555778309703, "clip_ratio/high_mean": 0.0055651484144618735, "clip_ratio/low_mean": 0.0026611327921273187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008226281323004514, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 880.3500366210938, "completions/mean_terminated_length": 704.7777709960938, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.17410478554666042, "epoch": 0.12457337883959044, "frac_reward_zero_std": 0.0, "grad_norm": 0.05094069242477417, "kl": 0.007169327407609671, "learning_rate": 7.883959044368601e-06, "loss": -0.1368, "num_tokens": 23554936.0, "reward": 0.7750000357627869, "reward_std": 0.9195176959037781, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.7481765747070312, "sampling/importance_sampling_ratio/mean": 0.4754231572151184, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3141660690307617, "sampling/sampling_logp_difference/mean": 0.016084710136055946, "step": 219, "step_time": 73.58871925366111 }, { "clip_ratio/high_max": 0.006806146935559809, "clip_ratio/high_mean": 0.0013612293842015788, "clip_ratio/low_mean": 0.002514648425858468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003875877824611962, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 903.0, "completions/mean_length": 867.125, "completions/mean_terminated_length": 541.3077392578125, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.12008063402026892, "epoch": 0.12514220705346984, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01900777779519558, "kl": 0.007204694964457303, "learning_rate": 7.878839590443686e-06, "loss": 0.1271, "num_tokens": 23662317.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.632622003555298, "sampling/importance_sampling_ratio/mean": 0.4765937328338623, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.516268253326416, "sampling/sampling_logp_difference/mean": 0.01305399090051651, "step": 220, "step_time": 73.3066552458331 }, { "clip_ratio/high_max": 0.017634662799537182, "clip_ratio/high_mean": 0.005853683134773746, "clip_ratio/low_mean": 0.0005126953146827873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006366378423990682, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 788.4000244140625, "completions/mean_terminated_length": 469.6470642089844, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.10520814172923565, "epoch": 0.12571103526734925, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.029522385448217392, "kl": 0.009171658428385854, "learning_rate": 7.87372013651877e-06, "loss": -0.0192, "num_tokens": 23770653.0, "reward": 0.824999988079071, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.6215386390686035, "sampling/importance_sampling_ratio/mean": 0.3758959174156189, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6962051391601562, "sampling/sampling_logp_difference/mean": 0.011749869212508202, "step": 221, "step_time": 74.86398750729859 }, { "clip_ratio/high_max": 0.004310056508984417, "clip_ratio/high_mean": 0.001066092896508053, "clip_ratio/low_mean": 0.001318359387369128, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0023844522947911173, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 1008.875, "completions/mean_terminated_length": 872.75, "completions/min_length": 735.0, "completions/min_terminated_length": 735.0, "entropy": 0.15634453482925892, "epoch": 0.12627986348122866, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.013019464910030365, "kl": 0.005126769392518327, "learning_rate": 7.868600682593857e-06, "loss": 0.0198, "num_tokens": 23884520.0, "reward": 0.17499999701976776, "reward_std": 0.5494752526283264, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 1.8785066604614258, "sampling/importance_sampling_ratio/mean": 0.30180591344833374, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4355850219726562, "sampling/sampling_logp_difference/mean": 0.012957543134689331, "step": 222, "step_time": 76.37112731579691 }, { "clip_ratio/high_max": 0.007022316451184452, "clip_ratio/high_mean": 0.0015073439572006464, "clip_ratio/low_mean": 0.0017822265544964466, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032895705371629447, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 952.7250366210938, "completions/mean_terminated_length": 616.7142944335938, "completions/min_length": 427.0, "completions/min_terminated_length": 427.0, "entropy": 0.13142311293631792, "epoch": 0.12684869169510807, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.011724601499736309, "kl": 0.0069442891399376094, "learning_rate": 7.863481228668943e-06, "loss": 0.0881, "num_tokens": 23996309.0, "reward": 0.3499999940395355, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.4941412210464478, "sampling/importance_sampling_ratio/mean": 0.3525750935077667, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.50477933883667, "sampling/sampling_logp_difference/mean": 0.012334357015788555, "step": 223, "step_time": 77.54497324093245 }, { "clip_ratio/high_max": 0.006653636402916163, "clip_ratio/high_mean": 0.0013307272747624665, "clip_ratio/low_mean": 0.002026367175858468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033570944506209344, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 911.0, "completions/mean_length": 991.6500244140625, "completions/mean_terminated_length": 765.2000122070312, "completions/min_length": 526.0, "completions/min_terminated_length": 526.0, "entropy": 0.12835585605353117, "epoch": 0.12741751990898748, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01390856597572565, "kl": 0.005729615571908653, "learning_rate": 7.858361774744027e-06, "loss": 0.0484, "num_tokens": 24104087.0, "reward": 0.25, "reward_std": 0.6698641180992126, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.4710984230041504, "sampling/importance_sampling_ratio/mean": 0.40819230675697327, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8378922939300537, "sampling/sampling_logp_difference/mean": 0.01223254669457674, "step": 224, "step_time": 73.85978105803952 }, { "clip_ratio/high_max": 0.013644171471241862, "clip_ratio/high_mean": 0.0034686244907788932, "clip_ratio/low_mean": 0.0017578125116415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005226436893281061, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 887.0, "completions/mean_length": 918.4500122070312, "completions/mean_terminated_length": 554.888916015625, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.12733947858214378, "epoch": 0.12798634812286688, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024540023878216743, "kl": 0.007721563626546413, "learning_rate": 7.853242320819112e-06, "loss": -0.0392, "num_tokens": 24206417.0, "reward": 0.4000000059604645, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.546436071395874, "sampling/importance_sampling_ratio/mean": 0.41263121366500854, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9408962726593018, "sampling/sampling_logp_difference/mean": 0.012597284279763699, "step": 225, "step_time": 71.25789882312529 }, { "clip_ratio/high_max": 0.010673425567802042, "clip_ratio/high_mean": 0.003003870020620525, "clip_ratio/low_mean": 0.0019775390574068297, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004981408957974054, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 931.4249877929688, "completions/mean_terminated_length": 653.7000122070312, "completions/min_length": 385.0, "completions/min_terminated_length": 385.0, "entropy": 0.12749398034065962, "epoch": 0.1285551763367463, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.05402739346027374, "kl": 0.007560098951216787, "learning_rate": 7.848122866894198e-06, "loss": -0.0932, "num_tokens": 24312882.0, "reward": 0.42500001192092896, "reward_std": 0.780778706073761, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.788198471069336, "sampling/importance_sampling_ratio/mean": 0.5570268630981445, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2112183570861816, "sampling/sampling_logp_difference/mean": 0.012152603827416897, "step": 226, "step_time": 73.07941823313013 }, { "clip_ratio/high_max": 0.014320729998871684, "clip_ratio/high_mean": 0.003921088398783468, "clip_ratio/low_mean": 0.001562500003274181, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005483588553033769, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 798.0, "completions/mean_length": 883.1749877929688, "completions/mean_terminated_length": 460.70001220703125, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.1485696081072092, "epoch": 0.1291240045506257, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018759172409772873, "kl": 0.008244874887168407, "learning_rate": 7.843003412969283e-06, "loss": -0.0215, "num_tokens": 24418217.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.0648305416107178, "sampling/importance_sampling_ratio/mean": 0.3519582748413086, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.865670084953308, "sampling/sampling_logp_difference/mean": 0.015227513387799263, "step": 227, "step_time": 72.56037319707684 }, { "clip_ratio/high_max": 0.01454170886427164, "clip_ratio/high_mean": 0.004223130468744785, "clip_ratio/low_mean": 0.0017822265508584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006005356990499422, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 803.2000122070312, "completions/mean_terminated_length": 622.5454711914062, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.0766590847633779, "epoch": 0.1296928327645051, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.05355221778154373, "kl": 0.010101649328134954, "learning_rate": 7.83788395904437e-06, "loss": -0.211, "num_tokens": 24517985.0, "reward": 1.100000023841858, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9902002811431885, "sampling/importance_sampling_ratio/mean": 0.5750003457069397, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7010703086853027, "sampling/sampling_logp_difference/mean": 0.010780408047139645, "step": 228, "step_time": 72.94544873177074 }, { "clip_ratio/high_max": 0.008251143910456449, "clip_ratio/high_mean": 0.0024103343748720363, "clip_ratio/low_mean": 0.0021972656577418093, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004607599956216291, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 820.1500244140625, "completions/mean_terminated_length": 544.3529663085938, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.13306852616369724, "epoch": 0.13026166097838454, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03481416404247284, "kl": 0.008617774408776313, "learning_rate": 7.832764505119454e-06, "loss": -0.0525, "num_tokens": 24628647.0, "reward": 0.800000011920929, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.98520565032959, "sampling/importance_sampling_ratio/mean": 0.430753231048584, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1872429847717285, "sampling/sampling_logp_difference/mean": 0.01531841792166233, "step": 229, "step_time": 75.80539654800668 }, { "clip_ratio/high_max": 0.016394891892559826, "clip_ratio/high_mean": 0.00558723314316012, "clip_ratio/low_mean": 0.001831054723879788, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007418287917971611, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 847.1000366210938, "completions/mean_terminated_length": 716.3478393554688, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.17141406424343586, "epoch": 0.13083048919226395, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019811533391475677, "kl": 0.009072268207091838, "learning_rate": 7.82764505119454e-06, "loss": 0.0624, "num_tokens": 24738547.0, "reward": 1.100000023841858, "reward_std": 0.9818872809410095, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.5597951412200928, "sampling/importance_sampling_ratio/mean": 0.30151301622390747, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.911716341972351, "sampling/sampling_logp_difference/mean": 0.01602300815284252, "step": 230, "step_time": 74.91220358968712 }, { "clip_ratio/high_max": 0.0071149267023429275, "clip_ratio/high_mean": 0.002704578801058233, "clip_ratio/low_mean": 0.0018066406264551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004511219420237467, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 944.375, "completions/mean_terminated_length": 734.45458984375, "completions/min_length": 420.0, "completions/min_terminated_length": 420.0, "entropy": 0.1186316292732954, "epoch": 0.13139931740614336, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.025767501443624496, "kl": 0.006493988796137273, "learning_rate": 7.822525597269625e-06, "loss": -0.0456, "num_tokens": 24850130.0, "reward": 0.45000001788139343, "reward_std": 0.7828285098075867, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.7497711181640625, "sampling/importance_sampling_ratio/mean": 0.40841323137283325, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4410276412963867, "sampling/sampling_logp_difference/mean": 0.01177195180207491, "step": 231, "step_time": 75.10258490499109 }, { "clip_ratio/high_max": 0.012331310717854649, "clip_ratio/high_mean": 0.003737318722414784, "clip_ratio/low_mean": 0.0012207031268189894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0049580217455513775, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 830.125, "completions/mean_terminated_length": 686.8261108398438, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.11704699508845806, "epoch": 0.13196814562002276, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020421037450432777, "kl": 0.009454409591853619, "learning_rate": 7.817406143344711e-06, "loss": -0.0778, "num_tokens": 24950991.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.926864743232727, "sampling/importance_sampling_ratio/mean": 0.3811269700527191, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.949875593185425, "sampling/sampling_logp_difference/mean": 0.013429728336632252, "step": 232, "step_time": 71.17432917514816 }, { "clip_ratio/high_max": 0.01446374476654455, "clip_ratio/high_mean": 0.005816372504341416, "clip_ratio/low_mean": 0.0015491658014070708, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007365538273006678, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 670.7250366210938, "completions/mean_terminated_length": 552.9666748046875, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.09299688134342432, "epoch": 0.13253697383390217, "frac_reward_zero_std": 0.0, "grad_norm": 0.027478262782096863, "kl": 0.010654031008016318, "learning_rate": 7.812286689419795e-06, "loss": 0.0779, "num_tokens": 25049604.0, "reward": 1.3250000476837158, "reward_std": 0.8589648008346558, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.8249644041061401, "sampling/importance_sampling_ratio/mean": 0.38663449883461, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2205722332000732, "sampling/sampling_logp_difference/mean": 0.012637975625693798, "step": 233, "step_time": 73.51946394983679 }, { "clip_ratio/high_max": 0.013609662652015686, "clip_ratio/high_mean": 0.0038749026571167633, "clip_ratio/low_mean": 0.0011718750247382559, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005046777747338638, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 764.0, "completions/mean_terminated_length": 476.631591796875, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.08935157861560583, "epoch": 0.13310580204778158, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024813666939735413, "kl": 0.013632044079713523, "learning_rate": 7.80716723549488e-06, "loss": 0.1711, "num_tokens": 25151988.0, "reward": 0.875, "reward_std": 0.9657599329948425, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.154015302658081, "sampling/importance_sampling_ratio/mean": 0.507725179195404, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.45988392829895, "sampling/sampling_logp_difference/mean": 0.012630745768547058, "step": 234, "step_time": 70.44966233288869 }, { "clip_ratio/high_max": 0.015003210515715182, "clip_ratio/high_mean": 0.0044148428132757545, "clip_ratio/low_mean": 0.0009277343797293724, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005342577234841883, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 846.0, "completions/mean_length": 875.8250122070312, "completions/mean_terminated_length": 600.6428833007812, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.10373501107096672, "epoch": 0.133674630261661, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016143111512064934, "kl": 0.012288030469790101, "learning_rate": 7.802047781569966e-06, "loss": 0.0436, "num_tokens": 25262757.0, "reward": 0.699999988079071, "reward_std": 0.9660917520523071, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.8681113719940186, "sampling/importance_sampling_ratio/mean": 0.37109917402267456, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.913029193878174, "sampling/sampling_logp_difference/mean": 0.01240553054958582, "step": 235, "step_time": 75.006754105445 }, { "clip_ratio/high_max": 0.01286942360457033, "clip_ratio/high_mean": 0.0034347494365647435, "clip_ratio/low_mean": 0.0023193359229480848, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.0057540853740647435, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 789.0, "completions/mean_length": 838.2000122070312, "completions/mean_terminated_length": 404.66668701171875, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.10179263260215521, "epoch": 0.1342434584755404, "frac_reward_zero_std": 0.0, "grad_norm": 0.026943376287817955, "kl": 0.01480306702433154, "learning_rate": 7.796928327645053e-06, "loss": -0.0064, "num_tokens": 25365837.0, "reward": 0.6000000238418579, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.8963871002197266, "sampling/importance_sampling_ratio/mean": 0.3995043933391571, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3716068267822266, "sampling/sampling_logp_difference/mean": 0.010810251347720623, "step": 236, "step_time": 72.13700455613434 }, { "clip_ratio/high_max": 0.007672892068512738, "clip_ratio/high_mean": 0.00264477489690762, "clip_ratio/low_mean": 0.0019287109153083293, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004573485755827278, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 921.6500244140625, "completions/mean_terminated_length": 614.6000366210938, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.05957447085529566, "epoch": 0.1348122866894198, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.034713517874479294, "kl": 0.008244893193477765, "learning_rate": 7.791808873720137e-06, "loss": -0.0056, "num_tokens": 25477151.0, "reward": 0.45000001788139343, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.9369888305664062, "sampling/importance_sampling_ratio/mean": 0.45684099197387695, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.708643913269043, "sampling/sampling_logp_difference/mean": 0.008081912994384766, "step": 237, "step_time": 74.93937698123045 }, { "clip_ratio/high_max": 0.014183727093040943, "clip_ratio/high_mean": 0.004515622276812792, "clip_ratio/low_mean": 0.00091952396905981, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005435146216768771, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 867.0, "completions/mean_length": 781.875, "completions/mean_terminated_length": 514.26318359375, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.10857179947197437, "epoch": 0.1353811149032992, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025112055242061615, "kl": 0.019754280569031835, "learning_rate": 7.786689419795222e-06, "loss": -0.078, "num_tokens": 25580986.0, "reward": 0.8500000238418579, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.677011489868164, "sampling/importance_sampling_ratio/mean": 0.4283462464809418, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3056559562683105, "sampling/sampling_logp_difference/mean": 0.014059643261134624, "step": 238, "step_time": 71.11152008897625 }, { "clip_ratio/high_max": 0.013691984582692385, "clip_ratio/high_mean": 0.004015467100543901, "clip_ratio/low_mean": 0.001811008938602754, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005826476000947878, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 829.0, "completions/mean_length": 860.25, "completions/mean_terminated_length": 520.1538696289062, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.12213132716715336, "epoch": 0.13594994311717862, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014331799000501633, "kl": 0.011540123785380274, "learning_rate": 7.781569965870308e-06, "loss": -0.0169, "num_tokens": 25688332.0, "reward": 0.6000000238418579, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.363734483718872, "sampling/importance_sampling_ratio/mean": 0.36355096101760864, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6709296703338623, "sampling/sampling_logp_difference/mean": 0.012466059997677803, "step": 239, "step_time": 74.76932828687131 }, { "clip_ratio/high_max": 0.006559525121701881, "clip_ratio/high_mean": 0.0019644620188046247, "clip_ratio/low_mean": 0.0009277343706344254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002892196425818838, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 953.0, "completions/mean_length": 899.5250244140625, "completions/mean_terminated_length": 712.8125, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 0.10842190496623516, "epoch": 0.13651877133105803, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.032118868082761765, "kl": 0.007325969636440277, "learning_rate": 7.776450511945393e-06, "loss": 0.1497, "num_tokens": 25779177.0, "reward": 0.6500000357627869, "reward_std": 0.8638019561767578, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.5371086597442627, "sampling/importance_sampling_ratio/mean": 0.40075549483299255, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7733325958251953, "sampling/sampling_logp_difference/mean": 0.011836696416139603, "step": 240, "step_time": 69.88661044789478 }, { "clip_ratio/high_max": 0.00933879183139652, "clip_ratio/high_mean": 0.0018677583721000701, "clip_ratio/low_mean": 0.002099609375363798, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003967367723817006, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 954.9750366210938, "completions/mean_terminated_length": 563.8333740234375, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.08058209856972098, "epoch": 0.13708759954493743, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02199586294591427, "kl": 0.006600711261853576, "learning_rate": 7.771331058020477e-06, "loss": 0.0427, "num_tokens": 25884624.0, "reward": 0.25, "reward_std": 0.6304251551628113, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.366314649581909, "sampling/importance_sampling_ratio/mean": 0.4854244291782379, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7640058994293213, "sampling/sampling_logp_difference/mean": 0.00955597497522831, "step": 241, "step_time": 73.85318827792071 }, { "clip_ratio/high_max": 0.014739581616595387, "clip_ratio/high_mean": 0.004393030350911431, "clip_ratio/low_mean": 0.0023251258753589354, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006718156218994409, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 800.0, "completions/mean_length": 862.8250122070312, "completions/mean_terminated_length": 486.75, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.11673788167536259, "epoch": 0.13765642775881684, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.017341667786240578, "kl": 0.012120778497774154, "learning_rate": 7.766211604095564e-06, "loss": 0.0059, "num_tokens": 25986257.0, "reward": 0.574999988079071, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.6959478855133057, "sampling/importance_sampling_ratio/mean": 0.36173513531684875, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.408395767211914, "sampling/sampling_logp_difference/mean": 0.014059486798942089, "step": 242, "step_time": 70.58190305135213 }, { "clip_ratio/high_max": 0.004378141544293612, "clip_ratio/high_mean": 0.0013563078828155994, "clip_ratio/low_mean": 0.0010053070836875122, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002361614999244921, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 833.9750366210938, "completions/mean_terminated_length": 678.5, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.08177896775305271, "epoch": 0.13822525597269625, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019862350076436996, "kl": 0.008634477038867772, "learning_rate": 7.761092150170648e-06, "loss": 0.0331, "num_tokens": 26081288.0, "reward": 0.875, "reward_std": 0.8825225830078125, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.0399229526519775, "sampling/importance_sampling_ratio/mean": 0.44657889008522034, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.593494176864624, "sampling/sampling_logp_difference/mean": 0.010954235680401325, "step": 243, "step_time": 70.15111380885355 }, { "clip_ratio/high_max": 0.0084360406617634, "clip_ratio/high_mean": 0.0023976860393304378, "clip_ratio/low_mean": 0.0026855468895519152, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.005083232914330438, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 921.0499877929688, "completions/mean_terminated_length": 649.6363525390625, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.11967638414353132, "epoch": 0.13879408418657566, "frac_reward_zero_std": 0.0, "grad_norm": 0.029226157814264297, "kl": 0.008169100154191256, "learning_rate": 7.755972696245734e-06, "loss": 0.0628, "num_tokens": 26187882.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.1937246322631836, "sampling/importance_sampling_ratio/mean": 0.5029372572898865, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.460130453109741, "sampling/sampling_logp_difference/mean": 0.012238726019859314, "step": 244, "step_time": 73.21524035953917 }, { "clip_ratio/high_max": 0.015451490762643516, "clip_ratio/high_mean": 0.005654014210449532, "clip_ratio/low_mean": 0.001230556721566245, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006884570815600455, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 798.0499877929688, "completions/mean_terminated_length": 631.0435180664062, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.09978162031620741, "epoch": 0.13936291240045506, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027944203466176987, "kl": 0.006912671378813684, "learning_rate": 7.750853242320819e-06, "loss": -0.0125, "num_tokens": 26280692.0, "reward": 1.0500000715255737, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.9260756969451904, "sampling/importance_sampling_ratio/mean": 0.42847442626953125, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.343630313873291, "sampling/sampling_logp_difference/mean": 0.011942907236516476, "step": 245, "step_time": 71.41309963609092 }, { "clip_ratio/high_max": 0.008006338903214782, "clip_ratio/high_mean": 0.0016012677951948717, "clip_ratio/low_mean": 0.001611328130820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003212595940567553, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 778.875, "completions/mean_terminated_length": 597.6956787109375, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.13184607215225697, "epoch": 0.13993174061433447, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018076732754707336, "kl": 0.019212467537727207, "learning_rate": 7.745733788395905e-06, "loss": 0.0027, "num_tokens": 26388527.0, "reward": 1.125, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9284141063690186, "sampling/importance_sampling_ratio/mean": 0.4206494688987732, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2066216468811035, "sampling/sampling_logp_difference/mean": 0.015096113085746765, "step": 246, "step_time": 75.18322028964758 }, { "clip_ratio/high_max": 0.01437890122178942, "clip_ratio/high_mean": 0.0042835091589950025, "clip_ratio/low_mean": 0.001684570284851361, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005968079436570406, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 918.8500366210938, "completions/mean_terminated_length": 700.4615478515625, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "entropy": 0.11035123374313116, "epoch": 0.14050056882821388, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027335893362760544, "kl": 0.008818653790513054, "learning_rate": 7.74061433447099e-06, "loss": 0.0085, "num_tokens": 26489225.0, "reward": 0.625, "reward_std": 0.9250779151916504, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.1803808212280273, "sampling/importance_sampling_ratio/mean": 0.42956143617630005, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.231696128845215, "sampling/sampling_logp_difference/mean": 0.011547795496881008, "step": 247, "step_time": 72.07750778994523 }, { "clip_ratio/high_max": 0.015108211897313595, "clip_ratio/high_mean": 0.006238310859771445, "clip_ratio/low_mean": 0.0013671875349245965, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007605498307384551, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 834.4500122070312, "completions/mean_terminated_length": 624.9473876953125, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "entropy": 0.1348674986511469, "epoch": 0.1410693970420933, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023193268105387688, "kl": 0.008370616706088185, "learning_rate": 7.735494880546074e-06, "loss": 0.0807, "num_tokens": 26597659.0, "reward": 0.8500000238418579, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.8723130226135254, "sampling/importance_sampling_ratio/mean": 0.39164409041404724, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7038114070892334, "sampling/sampling_logp_difference/mean": 0.014129278250038624, "step": 248, "step_time": 74.43162212730385 }, { "clip_ratio/high_max": 0.005241414764896035, "clip_ratio/high_mean": 0.0011632254754658788, "clip_ratio/low_mean": 0.0007812500116415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019444754580035806, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 925.0, "completions/mean_length": 901.7999877929688, "completions/mean_terminated_length": 674.857177734375, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.09012799896299839, "epoch": 0.1416382252559727, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.015255040489137173, "kl": 0.006134322728030384, "learning_rate": 7.73037542662116e-06, "loss": 0.0715, "num_tokens": 26700155.0, "reward": 0.625, "reward_std": 0.8969321250915527, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.2252986431121826, "sampling/importance_sampling_ratio/mean": 0.42109718918800354, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.656684398651123, "sampling/sampling_logp_difference/mean": 0.01045813225209713, "step": 249, "step_time": 74.57906603068113 }, { "clip_ratio/high_max": 0.014526775572448969, "clip_ratio/high_mean": 0.0070462247531395406, "clip_ratio/low_mean": 0.0015703673216194147, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008616592094767839, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 708.4249877929688, "completions/mean_terminated_length": 573.1785888671875, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.1076509254053235, "epoch": 0.1422070534698521, "frac_reward_zero_std": 0.0, "grad_norm": 0.043656639754772186, "kl": 0.01792366965673864, "learning_rate": 7.725255972696245e-06, "loss": -0.033, "num_tokens": 26804668.0, "reward": 1.350000023841858, "reward_std": 0.9212588667869568, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.7241439819335938, "sampling/importance_sampling_ratio/mean": 0.6084960699081421, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9494400024414062, "sampling/sampling_logp_difference/mean": 0.01332577969878912, "step": 250, "step_time": 72.87908109393902 }, { "clip_ratio/high_max": 0.01523978030309081, "clip_ratio/high_mean": 0.006247628771234304, "clip_ratio/low_mean": 0.0019531250072759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008200753829441965, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 797.875, "completions/mean_terminated_length": 612.8636474609375, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "entropy": 0.09772372618317604, "epoch": 0.1427758816837315, "frac_reward_zero_std": 0.0, "grad_norm": 0.032864753156900406, "kl": 0.010427681438159198, "learning_rate": 7.720136518771332e-06, "loss": -0.1027, "num_tokens": 26911439.0, "reward": 1.0, "reward_std": 0.9607689380645752, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.8982861042022705, "sampling/importance_sampling_ratio/mean": 0.5244356393814087, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.574511766433716, "sampling/sampling_logp_difference/mean": 0.01213865540921688, "step": 251, "step_time": 72.6709643539507 }, { "clip_ratio/high_max": 0.011934182257391512, "clip_ratio/high_mean": 0.0034529757540440187, "clip_ratio/low_mean": 0.0015603667070536176, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005013342364691198, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 818.0, "completions/mean_length": 802.375, "completions/mean_terminated_length": 580.75, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "entropy": 0.11470470856875181, "epoch": 0.14334470989761092, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015521145425736904, "kl": 0.009595884650480002, "learning_rate": 7.715017064846416e-06, "loss": -0.0251, "num_tokens": 27014134.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.7652034759521484, "sampling/importance_sampling_ratio/mean": 0.49337419867515564, "sampling/importance_sampling_ratio/min": 0.004346833098679781, "sampling/sampling_logp_difference/max": 2.7988953590393066, "sampling/sampling_logp_difference/mean": 0.01396979670971632, "step": 252, "step_time": 73.9836954078637 }, { "clip_ratio/high_max": 0.007802460837410763, "clip_ratio/high_mean": 0.0018767310612020083, "clip_ratio/low_mean": 0.001977539071958745, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003854270209558308, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 931.8500366210938, "completions/mean_terminated_length": 563.25, "completions/min_length": 445.0, "completions/min_terminated_length": 445.0, "entropy": 0.06322708120569587, "epoch": 0.14391353811149032, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.016784192994236946, "kl": 0.0058574696304276586, "learning_rate": 7.709897610921502e-06, "loss": 0.0489, "num_tokens": 27121816.0, "reward": 0.375, "reward_std": 0.7741827964782715, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.259315252304077, "sampling/importance_sampling_ratio/mean": 0.3973914086818695, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2219066619873047, "sampling/sampling_logp_difference/mean": 0.008000651374459267, "step": 253, "step_time": 73.08577111805789 }, { "clip_ratio/high_max": 0.010898657608777285, "clip_ratio/high_mean": 0.004243607560056262, "clip_ratio/low_mean": 0.002351428694964852, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0065950361313298345, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 827.5, "completions/mean_terminated_length": 743.2857666015625, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "entropy": 0.11931702867150307, "epoch": 0.14448236632536973, "frac_reward_zero_std": 0.0, "grad_norm": 0.04015198349952698, "kl": 0.007930464576929808, "learning_rate": 7.704778156996587e-06, "loss": -0.0872, "num_tokens": 27224796.0, "reward": 1.225000023841858, "reward_std": 0.8911962509155273, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.9633898735046387, "sampling/importance_sampling_ratio/mean": 0.6839975118637085, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.846318006515503, "sampling/sampling_logp_difference/mean": 0.01326075941324234, "step": 254, "step_time": 73.15012093982659 }, { "clip_ratio/high_max": 0.00886647988227196, "clip_ratio/high_mean": 0.002277840168972034, "clip_ratio/low_mean": 0.0010253906530124368, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033032307983376086, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 809.9750366210938, "completions/mean_terminated_length": 573.4210815429688, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.09120209887623787, "epoch": 0.14505119453924914, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03272340074181557, "kl": 0.008876771898940206, "learning_rate": 7.699658703071673e-06, "loss": 0.0613, "num_tokens": 27331139.0, "reward": 0.875, "reward_std": 0.9657599329948425, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.7655587196350098, "sampling/importance_sampling_ratio/mean": 0.563147783279419, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.658601760864258, "sampling/sampling_logp_difference/mean": 0.011512886732816696, "step": 255, "step_time": 73.28708991012536 }, { "clip_ratio/high_max": 0.013585373875685036, "clip_ratio/high_mean": 0.003152715813484974, "clip_ratio/low_mean": 0.0007812500134605216, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003933965839678422, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 967.0, "completions/mean_length": 802.7000122070312, "completions/mean_terminated_length": 532.2222290039062, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.14633511006832123, "epoch": 0.14562002275312855, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.03086892142891884, "kl": 0.011712794192135334, "learning_rate": 7.694539249146758e-06, "loss": 0.0443, "num_tokens": 27433663.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.3022515773773193, "sampling/importance_sampling_ratio/mean": 0.6857194900512695, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.001262903213501, "sampling/sampling_logp_difference/mean": 0.015519793145358562, "step": 256, "step_time": 73.98756484827027 }, { "clip_ratio/high_max": 0.004566803254419938, "clip_ratio/high_mean": 0.0009133606581599452, "clip_ratio/low_mean": 0.0008300781246362021, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017434387336834334, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 655.0, "completions/mean_length": 866.1749877929688, "completions/mean_terminated_length": 497.91668701171875, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.07494701026007533, "epoch": 0.14618885096700796, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01706530712544918, "kl": 0.005892487679375336, "learning_rate": 7.689419795221843e-06, "loss": 0.0294, "num_tokens": 27543022.0, "reward": 0.574999988079071, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.349567413330078, "sampling/importance_sampling_ratio/mean": 0.4868372976779938, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.273876667022705, "sampling/sampling_logp_difference/mean": 0.010514731518924236, "step": 257, "step_time": 73.30244172364473 }, { "clip_ratio/high_max": 0.011358909483533353, "clip_ratio/high_mean": 0.003938560839742422, "clip_ratio/low_mean": 0.0015669568674638867, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005505517590790987, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 739.9249877929688, "completions/mean_terminated_length": 586.9615478515625, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.10747814830392599, "epoch": 0.14675767918088736, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02632533758878708, "kl": 0.0180025162990205, "learning_rate": 7.684300341296929e-06, "loss": -0.0295, "num_tokens": 27648843.0, "reward": 1.100000023841858, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.697058916091919, "sampling/importance_sampling_ratio/mean": 0.5150572061538696, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7711358070373535, "sampling/sampling_logp_difference/mean": 0.012589867226779461, "step": 258, "step_time": 73.69647147506475 }, { "clip_ratio/high_max": 0.012744813575409353, "clip_ratio/high_mean": 0.003205843531759456, "clip_ratio/low_mean": 0.0033691406424622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006574984174221754, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 957.375, "completions/mean_terminated_length": 643.2857666015625, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.1620920803397894, "epoch": 0.14732650739476677, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.012021453119814396, "kl": 0.00814844120759517, "learning_rate": 7.679180887372013e-06, "loss": 0.104, "num_tokens": 27754554.0, "reward": 0.3499999940395355, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.8257288932800293, "sampling/importance_sampling_ratio/mean": 0.3485647737979889, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.78386390209198, "sampling/sampling_logp_difference/mean": 0.014758038334548473, "step": 259, "step_time": 74.27462640404701 }, { "clip_ratio/high_max": 0.015992380562238395, "clip_ratio/high_mean": 0.005217067955527455, "clip_ratio/low_mean": 0.0016113281235448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006828396231867373, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 832.2999877929688, "completions/mean_terminated_length": 598.0, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.12196646071970463, "epoch": 0.14789533560864618, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020534489303827286, "kl": 0.008578475215472281, "learning_rate": 7.674061433447098e-06, "loss": 0.0565, "num_tokens": 27870222.0, "reward": 0.75, "reward_std": 0.8987170457839966, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.8917043209075928, "sampling/importance_sampling_ratio/mean": 0.6295068264007568, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9836026430130005, "sampling/sampling_logp_difference/mean": 0.01373532135039568, "step": 260, "step_time": 77.83233724324964 }, { "clip_ratio/high_max": 0.01746326091233641, "clip_ratio/high_mean": 0.006799503447837196, "clip_ratio/low_mean": 0.0017089843895519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00850848777918145, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 959.0, "completions/mean_length": 824.3250122070312, "completions/mean_terminated_length": 580.2777709960938, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "entropy": 0.1361439824104309, "epoch": 0.14846416382252559, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.010090206749737263, "kl": 0.0076145787024870515, "learning_rate": 7.668941979522184e-06, "loss": -0.0249, "num_tokens": 27984891.0, "reward": 0.800000011920929, "reward_std": 0.9391758441925049, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 0.9914524555206299, "sampling/importance_sampling_ratio/mean": 0.24859316647052765, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9861170053482056, "sampling/sampling_logp_difference/mean": 0.014133612625300884, "step": 261, "step_time": 75.35564356739633 }, { "clip_ratio/high_max": 0.009550673625199124, "clip_ratio/high_mean": 0.004090716094651725, "clip_ratio/low_mean": 0.0020751953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0061659115890506655, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 909.0, "completions/mean_length": 856.4500122070312, "completions/mean_terminated_length": 651.6666870117188, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.08215265022590756, "epoch": 0.149032992036405, "frac_reward_zero_std": 0.0, "grad_norm": 0.03740141913294792, "kl": 0.007018008443992585, "learning_rate": 7.66382252559727e-06, "loss": 0.0625, "num_tokens": 28090493.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.716254472732544, "sampling/importance_sampling_ratio/mean": 0.5170547366142273, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1597046852111816, "sampling/sampling_logp_difference/mean": 0.010178869590163231, "step": 262, "step_time": 71.93199446192011 }, { "clip_ratio/high_max": 0.011004135012626648, "clip_ratio/high_mean": 0.00313985715911258, "clip_ratio/low_mean": 0.0014057710286579095, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00454562806407921, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 909.0499877929688, "completions/mean_terminated_length": 670.3077392578125, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.08901164401322603, "epoch": 0.1496018202502844, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03777951002120972, "kl": 0.006291892088484019, "learning_rate": 7.658703071672355e-06, "loss": -0.0077, "num_tokens": 28201623.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.6094889640808105, "sampling/importance_sampling_ratio/mean": 0.6439194679260254, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2428321838378906, "sampling/sampling_logp_difference/mean": 0.010568959638476372, "step": 263, "step_time": 75.96032461430877 }, { "clip_ratio/high_max": 0.016780529753305018, "clip_ratio/high_mean": 0.005147448042407632, "clip_ratio/low_mean": 0.001879882809589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0070273308956529945, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 745.8250122070312, "completions/mean_terminated_length": 518.227294921875, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 0.14224238507449627, "epoch": 0.15017064846416384, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03267233073711395, "kl": 0.013067251828033477, "learning_rate": 7.65358361774744e-06, "loss": -0.0605, "num_tokens": 28302320.0, "reward": 0.9750000238418579, "reward_std": 0.9469926357269287, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.917098045349121, "sampling/importance_sampling_ratio/mean": 0.4096698760986328, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7741150856018066, "sampling/sampling_logp_difference/mean": 0.014962988905608654, "step": 264, "step_time": 71.06987845338881 }, { "clip_ratio/high_max": 0.009295408497564495, "clip_ratio/high_mean": 0.0024123270632117055, "clip_ratio/low_mean": 0.001660156252910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004072483396157622, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 940.6749877929688, "completions/mean_terminated_length": 690.7000122070312, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.10694098193198442, "epoch": 0.15073947667804324, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.017874587327241898, "kl": 0.005342439690139145, "learning_rate": 7.648464163822526e-06, "loss": 0.09, "num_tokens": 28417715.0, "reward": 0.4749999940395355, "reward_std": 0.8469401597976685, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 1.5533349514007568, "sampling/importance_sampling_ratio/mean": 0.4245632290840149, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6565072536468506, "sampling/sampling_logp_difference/mean": 0.011392340064048767, "step": 265, "step_time": 77.36032964126207 }, { "clip_ratio/high_max": 0.01257160643581301, "clip_ratio/high_mean": 0.003435527381952852, "clip_ratio/low_mean": 0.0008056640799622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004241191447363235, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 872.0, "completions/mean_length": 867.4249877929688, "completions/mean_terminated_length": 454.6363830566406, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.05930829653516412, "epoch": 0.15130830489192265, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.029545137658715248, "kl": 0.007392170373350382, "learning_rate": 7.64334470989761e-06, "loss": -0.0509, "num_tokens": 28525044.0, "reward": 0.4749999940395355, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.457080364227295, "sampling/importance_sampling_ratio/mean": 0.6370745897293091, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.971482753753662, "sampling/sampling_logp_difference/mean": 0.007897351868450642, "step": 266, "step_time": 73.65091797034256 }, { "clip_ratio/high_max": 0.01548442686907947, "clip_ratio/high_mean": 0.0069007136626169086, "clip_ratio/low_mean": 0.0017822265508584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008682940213475376, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 969.0, "completions/mean_length": 779.0250244140625, "completions/mean_terminated_length": 578.5909423828125, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.12264874670654535, "epoch": 0.15187713310580206, "frac_reward_zero_std": 0.0, "grad_norm": 0.030589060857892036, "kl": 0.01141057925997302, "learning_rate": 7.638225255972697e-06, "loss": 0.0292, "num_tokens": 28629557.0, "reward": 1.024999976158142, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.660053253173828, "sampling/importance_sampling_ratio/mean": 0.38107189536094666, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1528568267822266, "sampling/sampling_logp_difference/mean": 0.013083639554679394, "step": 267, "step_time": 72.83095046179369 }, { "clip_ratio/high_max": 0.016547138104215264, "clip_ratio/high_mean": 0.004985347128240392, "clip_ratio/low_mean": 0.002661132821231149, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0076464799931272864, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 845.2000122070312, "completions/mean_terminated_length": 603.2941284179688, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.09929057816043496, "epoch": 0.15244596131968147, "frac_reward_zero_std": 0.0, "grad_norm": 0.03977516293525696, "kl": 0.008944344182964414, "learning_rate": 7.633105802047781e-06, "loss": -0.1485, "num_tokens": 28735317.0, "reward": 0.7750000357627869, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.5244152545928955, "sampling/importance_sampling_ratio/mean": 0.48439884185791016, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.700671672821045, "sampling/sampling_logp_difference/mean": 0.011032864451408386, "step": 268, "step_time": 72.19592938292772 }, { "clip_ratio/high_max": 0.013238107319921255, "clip_ratio/high_mean": 0.005610854277620092, "clip_ratio/low_mean": 0.001440429690774181, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007051283959299326, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 923.0, "completions/mean_length": 803.25, "completions/mean_terminated_length": 622.6363525390625, "completions/min_length": 425.0, "completions/min_terminated_length": 425.0, "entropy": 0.08778100833296776, "epoch": 0.15301478953356087, "frac_reward_zero_std": 0.0, "grad_norm": 0.03923669084906578, "kl": 0.016050245729275048, "learning_rate": 7.627986348122867e-06, "loss": -0.0643, "num_tokens": 28839439.0, "reward": 1.0750000476837158, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057363510131836, "sampling/importance_sampling_ratio/max": 2.811030864715576, "sampling/importance_sampling_ratio/mean": 0.41873255372047424, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.150665760040283, "sampling/sampling_logp_difference/mean": 0.012148737907409668, "step": 269, "step_time": 74.35335796675645 }, { "clip_ratio/high_max": 0.01385816908441484, "clip_ratio/high_mean": 0.00588475281256251, "clip_ratio/low_mean": 0.0023445575934601948, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00822931039147079, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 958.0, "completions/mean_length": 772.25, "completions/mean_terminated_length": 586.1739501953125, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.13182047195732594, "epoch": 0.15358361774744028, "frac_reward_zero_std": 0.0, "grad_norm": 0.026105109602212906, "kl": 0.00954615673981607, "learning_rate": 7.622866894197952e-06, "loss": 0.0507, "num_tokens": 28948937.0, "reward": 1.024999976158142, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.7128828763961792, "sampling/importance_sampling_ratio/mean": 0.3875119686126709, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4953339099884033, "sampling/sampling_logp_difference/mean": 0.013846154324710369, "step": 270, "step_time": 75.96475408249535 }, { "clip_ratio/high_max": 0.0021596375154331326, "clip_ratio/high_mean": 0.00043192750308662653, "clip_ratio/low_mean": 0.0003906250112777343, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008225525143643608, "completions/clipped_ratio": 0.949999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 1005.1749877929688, "completions/mean_terminated_length": 647.5, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.10304925497621298, "epoch": 0.1541524459613197, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.014059437438845634, "kl": 0.004527182201854885, "learning_rate": 7.617747440273038e-06, "loss": -0.02, "num_tokens": 29060568.0, "reward": 0.10000000149011612, "reward_std": 0.4414428472518921, "rewards/format_reward/mean": 0.05000000074505806, "rewards/format_reward/std": 0.22072142362594604, "rewards/gpt_judge_reward/mean": 0.05000000074505806, "rewards/gpt_judge_reward/std": 0.22072142362594604, "sampling/importance_sampling_ratio/max": 2.829406261444092, "sampling/importance_sampling_ratio/mean": 0.6172451376914978, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8378846645355225, "sampling/sampling_logp_difference/mean": 0.010558657348155975, "step": 271, "step_time": 74.56911098421551 }, { "clip_ratio/high_max": 0.005716365645639598, "clip_ratio/high_mean": 0.001385208626743406, "clip_ratio/low_mean": 0.0025146484695142135, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003899857052601874, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 905.4249877929688, "completions/mean_terminated_length": 659.1538696289062, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "entropy": 0.17374329827725887, "epoch": 0.1547212741751991, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016307009384036064, "kl": 0.006759762531146407, "learning_rate": 7.612627986348122e-06, "loss": 0.0465, "num_tokens": 29169561.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.0522990226745605, "sampling/importance_sampling_ratio/mean": 0.3577660620212555, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1742916107177734, "sampling/sampling_logp_difference/mean": 0.015328067354857922, "step": 272, "step_time": 72.19966564304195 }, { "clip_ratio/high_max": 0.007350335275987163, "clip_ratio/high_mean": 0.001959429338967311, "clip_ratio/low_mean": 0.0018437736871419474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038032030424801633, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 870.8500366210938, "completions/mean_terminated_length": 717.7000122070312, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "entropy": 0.08786544064059854, "epoch": 0.1552901023890785, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027747265994548798, "kl": 0.005741560540627688, "learning_rate": 7.607508532423209e-06, "loss": -0.0331, "num_tokens": 29270763.0, "reward": 0.6500000357627869, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.3957247734069824, "sampling/importance_sampling_ratio/mean": 0.4935995042324066, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0374839305877686, "sampling/sampling_logp_difference/mean": 0.010612628422677517, "step": 273, "step_time": 73.25635662605055 }, { "clip_ratio/high_max": 0.008249160076957196, "clip_ratio/high_mean": 0.001649832003749907, "clip_ratio/low_mean": 0.0002685546933207661, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001918386697070673, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 888.0, "completions/mean_length": 885.4500122070312, "completions/mean_terminated_length": 628.1428833007812, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "entropy": 0.14877862110733986, "epoch": 0.1558589306029579, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.023309456184506416, "kl": 0.006997820222750306, "learning_rate": 7.602389078498294e-06, "loss": 0.0954, "num_tokens": 29377357.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.210362195968628, "sampling/importance_sampling_ratio/mean": 0.5364020466804504, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.334530830383301, "sampling/sampling_logp_difference/mean": 0.014788690023124218, "step": 274, "step_time": 74.44437422812916 }, { "clip_ratio/high_max": 0.012137405574321747, "clip_ratio/high_mean": 0.002884933172026649, "clip_ratio/low_mean": 0.0012451172005967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0041300503216916695, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 907.0, "completions/mean_length": 801.9249877929688, "completions/mean_terminated_length": 501.4705810546875, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.11059023533016443, "epoch": 0.15642775881683732, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0196888018399477, "kl": 0.00886237103259191, "learning_rate": 7.5972696245733795e-06, "loss": -0.0281, "num_tokens": 29487082.0, "reward": 0.824999988079071, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.7255985736846924, "sampling/importance_sampling_ratio/mean": 0.5148410797119141, "sampling/importance_sampling_ratio/min": 0.012215142138302326, "sampling/sampling_logp_difference/max": 1.8762855529785156, "sampling/sampling_logp_difference/mean": 0.01282077468931675, "step": 275, "step_time": 74.08551249210723 }, { "clip_ratio/high_max": 0.006722940015606582, "clip_ratio/high_mean": 0.0013445880176732317, "clip_ratio/low_mean": 0.00031738281177240424, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001661970829445636, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 833.25, "completions/mean_terminated_length": 660.6666870117188, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "entropy": 0.11366697866469622, "epoch": 0.15699658703071673, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.02739519253373146, "kl": 0.0073645494994707406, "learning_rate": 7.592150170648464e-06, "loss": 0.0395, "num_tokens": 29590020.0, "reward": 0.9750000238418579, "reward_std": 0.9736924767494202, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.704249143600464, "sampling/importance_sampling_ratio/mean": 0.7061160802841187, "sampling/importance_sampling_ratio/min": 0.003349899547174573, "sampling/sampling_logp_difference/max": 1.752948522567749, "sampling/sampling_logp_difference/mean": 0.012951555661857128, "step": 276, "step_time": 70.30657426430844 }, { "clip_ratio/high_max": 0.008632784825749695, "clip_ratio/high_mean": 0.0027248245023656636, "clip_ratio/low_mean": 0.0020996093735448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004824433824978769, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 795.0, "completions/mean_length": 770.125, "completions/mean_terminated_length": 516.25, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.1165951881557703, "epoch": 0.15756541524459614, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.021982694044709206, "kl": 0.010940322652459145, "learning_rate": 7.5870307167235495e-06, "loss": 0.0977, "num_tokens": 29697545.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.0350427627563477, "sampling/importance_sampling_ratio/mean": 0.44290170073509216, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9117555618286133, "sampling/sampling_logp_difference/mean": 0.013552272692322731, "step": 277, "step_time": 75.3134798379615 }, { "clip_ratio/high_max": 0.00845670752460137, "clip_ratio/high_mean": 0.001691341502009891, "clip_ratio/low_mean": 0.0023942036787047982, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004085545049747452, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 953.0, "completions/mean_length": 947.4000244140625, "completions/mean_terminated_length": 641.0, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "entropy": 0.14795407094061375, "epoch": 0.15813424345847554, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016613846644759178, "kl": 0.006590417353436351, "learning_rate": 7.581911262798635e-06, "loss": 0.0406, "num_tokens": 29808961.0, "reward": 0.3499999940395355, "reward_std": 0.7355444431304932, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.348522186279297, "sampling/importance_sampling_ratio/mean": 0.42443418502807617, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4750970602035522, "sampling/sampling_logp_difference/mean": 0.01407435443252325, "step": 278, "step_time": 75.2596417022869 }, { "clip_ratio/high_max": 0.007419191941153258, "clip_ratio/high_mean": 0.0021933372190687805, "clip_ratio/low_mean": 0.001904296877910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004097634082427248, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 926.7000122070312, "completions/mean_terminated_length": 724.6154174804688, "completions/min_length": 418.0, "completions/min_terminated_length": 418.0, "entropy": 0.13303519506007433, "epoch": 0.15870307167235495, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030778080224990845, "kl": 0.00601092743454501, "learning_rate": 7.576791808873721e-06, "loss": 0.0334, "num_tokens": 29905949.0, "reward": 0.6000000238418579, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.7622692584991455, "sampling/importance_sampling_ratio/mean": 0.4786587357521057, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.966679573059082, "sampling/sampling_logp_difference/mean": 0.012093308381736279, "step": 279, "step_time": 70.82685098354705 }, { "clip_ratio/high_max": 0.010765964223537594, "clip_ratio/high_mean": 0.002992596768308431, "clip_ratio/low_mean": 0.0018554687485448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004848065524129197, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 916.0, "completions/mean_length": 817.7250366210938, "completions/mean_terminated_length": 611.4500122070312, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.11446757055819035, "epoch": 0.15927189988623436, "frac_reward_zero_std": 0.0, "grad_norm": 0.024521294981241226, "kl": 0.009208559582475573, "learning_rate": 7.571672354948806e-06, "loss": -0.0079, "num_tokens": 30016810.0, "reward": 0.7750000357627869, "reward_std": 0.8619447350502014, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.277343511581421, "sampling/importance_sampling_ratio/mean": 0.4651360511779785, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8365569114685059, "sampling/sampling_logp_difference/mean": 0.013153046369552612, "step": 280, "step_time": 76.09064763179049 }, { "clip_ratio/high_max": 0.014682261913549155, "clip_ratio/high_mean": 0.0072380708588752896, "clip_ratio/low_mean": 0.0014910051686456427, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008729076129384339, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 736.625, "completions/mean_terminated_length": 564.2000122070312, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.13147331960499287, "epoch": 0.15984072810011377, "frac_reward_zero_std": 0.0, "grad_norm": 0.023468803614377975, "kl": 0.009621483972296119, "learning_rate": 7.566552901023891e-06, "loss": -0.0697, "num_tokens": 30120451.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 1.5500038862228394, "sampling/importance_sampling_ratio/mean": 0.3829790949821472, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.785353422164917, "sampling/sampling_logp_difference/mean": 0.014514184556901455, "step": 281, "step_time": 73.33960956288502 }, { "clip_ratio/high_max": 0.014544218662194908, "clip_ratio/high_mean": 0.005922804935835302, "clip_ratio/low_mean": 0.002018809893343132, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007941614836454391, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 805.7000122070312, "completions/mean_terminated_length": 608.1904907226562, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.1162009509280324, "epoch": 0.16040955631399317, "frac_reward_zero_std": 0.0, "grad_norm": 0.02680888958275318, "kl": 0.009620041586458683, "learning_rate": 7.561433447098977e-06, "loss": 0.0721, "num_tokens": 30225495.0, "reward": 0.925000011920929, "reward_std": 0.9442809820175171, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.77935528755188, "sampling/importance_sampling_ratio/mean": 0.5145425796508789, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3926291465759277, "sampling/sampling_logp_difference/mean": 0.013088256120681763, "step": 282, "step_time": 73.84387382306159 }, { "clip_ratio/high_max": 0.01156851794803515, "clip_ratio/high_mean": 0.0044125191125203855, "clip_ratio/low_mean": 0.0017611602197575849, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0061736792558804154, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 857.6749877929688, "completions/mean_terminated_length": 654.388916015625, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 0.14309369027614594, "epoch": 0.16097838452787258, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013238148763775826, "kl": 0.00797528144903481, "learning_rate": 7.556313993174061e-06, "loss": 0.0289, "num_tokens": 30332170.0, "reward": 0.8500000238418579, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.0773252248764038, "sampling/importance_sampling_ratio/mean": 0.22017419338226318, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3927009105682373, "sampling/sampling_logp_difference/mean": 0.014523453079164028, "step": 283, "step_time": 73.02293043001555 }, { "clip_ratio/high_max": 0.01156006520614028, "clip_ratio/high_mean": 0.0025128454144578427, "clip_ratio/low_mean": 0.0007925372374302242, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033053826773539186, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 888.8500366210938, "completions/mean_terminated_length": 706.0, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.13794054742902517, "epoch": 0.161547212741752, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.023869983851909637, "kl": 0.009062803001143038, "learning_rate": 7.551194539249147e-06, "loss": 0.0057, "num_tokens": 30441812.0, "reward": 0.824999988079071, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.6988227367401123, "sampling/importance_sampling_ratio/mean": 0.5667681694030762, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.143710136413574, "sampling/sampling_logp_difference/mean": 0.014789178967475891, "step": 284, "step_time": 75.62849564501084 }, { "clip_ratio/high_max": 0.008754893788136542, "clip_ratio/high_mean": 0.002738320719799958, "clip_ratio/low_mean": 0.0017578124843566911, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004496133362408727, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 878.0, "completions/mean_length": 909.2250366210938, "completions/mean_terminated_length": 513.888916015625, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.07006356632336974, "epoch": 0.1621160409556314, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021802380681037903, "kl": 0.01938228460494429, "learning_rate": 7.546075085324232e-06, "loss": 0.0107, "num_tokens": 30551141.0, "reward": 0.45000001788139343, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.5513815879821777, "sampling/importance_sampling_ratio/mean": 0.3604770004749298, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.804471969604492, "sampling/sampling_logp_difference/mean": 0.009392069652676582, "step": 285, "step_time": 74.90713534713723 }, { "clip_ratio/high_max": 0.010460942634381354, "clip_ratio/high_mean": 0.0028448497178032994, "clip_ratio/low_mean": 0.0024990935962705407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00534394322312437, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 935.0, "completions/mean_length": 886.1000366210938, "completions/mean_terminated_length": 630.0, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 0.114860606379807, "epoch": 0.1626848691695108, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01833370327949524, "kl": 0.006819179281592369, "learning_rate": 7.5409556313993176e-06, "loss": 0.0801, "num_tokens": 30662433.0, "reward": 0.574999988079071, "reward_std": 0.8439072966575623, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.501823902130127, "sampling/importance_sampling_ratio/mean": 0.346212238073349, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.799111843109131, "sampling/sampling_logp_difference/mean": 0.01224692165851593, "step": 286, "step_time": 75.74366756551899 }, { "clip_ratio/high_max": 0.017426203936338425, "clip_ratio/high_mean": 0.00643559213494882, "clip_ratio/low_mean": 0.002864898255211301, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009300490317400545, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 817.625, "completions/mean_terminated_length": 648.7727661132812, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.15303605049848557, "epoch": 0.1632536973833902, "frac_reward_zero_std": 0.0, "grad_norm": 0.051711179316043854, "kl": 0.01566260342951864, "learning_rate": 7.535836177474402e-06, "loss": 0.04, "num_tokens": 30772770.0, "reward": 1.0, "reward_std": 0.96076899766922, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.982903480529785, "sampling/importance_sampling_ratio/mean": 0.6717119216918945, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.449842691421509, "sampling/sampling_logp_difference/mean": 0.01567993126809597, "step": 287, "step_time": 74.4516234791372 }, { "clip_ratio/high_max": 0.008640724350698292, "clip_ratio/high_mean": 0.0020633907843148336, "clip_ratio/low_mean": 0.0014404296925931703, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003503820422338322, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 872.0250244140625, "completions/mean_terminated_length": 666.4117431640625, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.08389988914132118, "epoch": 0.16382252559726962, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02603362314403057, "kl": 0.007019691343884915, "learning_rate": 7.5307167235494884e-06, "loss": 0.0536, "num_tokens": 30882891.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.557936668395996, "sampling/importance_sampling_ratio/mean": 0.5004073977470398, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.732835292816162, "sampling/sampling_logp_difference/mean": 0.010680892504751682, "step": 288, "step_time": 75.13269229093567 }, { "clip_ratio/high_max": 0.01562811341136694, "clip_ratio/high_mean": 0.005103788193082437, "clip_ratio/low_mean": 0.001904296877910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007008085114648566, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 706.125, "completions/mean_terminated_length": 515.3999633789062, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.14555106777697802, "epoch": 0.16439135381114903, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03840428963303566, "kl": 0.008120744314510375, "learning_rate": 7.525597269624574e-06, "loss": 0.1411, "num_tokens": 30988200.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 1.9484765529632568, "sampling/importance_sampling_ratio/mean": 0.4311501383781433, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.935107707977295, "sampling/sampling_logp_difference/mean": 0.016036896035075188, "step": 289, "step_time": 72.9554665570613 }, { "clip_ratio/high_max": 0.01331270404625684, "clip_ratio/high_mean": 0.004598395840730518, "clip_ratio/low_mean": 0.0011718750247382559, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005770270712673664, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 797.2750244140625, "completions/mean_terminated_length": 629.6956787109375, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.13050401769578457, "epoch": 0.16496018202502843, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.027207642793655396, "kl": 0.007318676391150802, "learning_rate": 7.520477815699659e-06, "loss": 0.0987, "num_tokens": 31099779.0, "reward": 0.8500000238418579, "reward_std": 0.8335896730422974, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.946099281311035, "sampling/importance_sampling_ratio/mean": 0.6347830891609192, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9799249172210693, "sampling/sampling_logp_difference/mean": 0.01413805317133665, "step": 290, "step_time": 73.86631051078439 }, { "clip_ratio/high_max": 0.00994805694790557, "clip_ratio/high_mean": 0.0031464734784094617, "clip_ratio/low_mean": 0.0006103515697759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003756825055461377, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 916.9750366210938, "completions/mean_terminated_length": 772.1764526367188, "completions/min_length": 485.0, "completions/min_terminated_length": 485.0, "entropy": 0.07035179389640689, "epoch": 0.16552901023890784, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03463371843099594, "kl": 0.00559999531833455, "learning_rate": 7.515358361774744e-06, "loss": 0.0484, "num_tokens": 31209730.0, "reward": 0.6000000238418579, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.8987607955932617, "sampling/importance_sampling_ratio/mean": 0.6644327044487, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.656148910522461, "sampling/sampling_logp_difference/mean": 0.00934422668069601, "step": 291, "step_time": 76.1433660229668 }, { "clip_ratio/high_max": 0.0028880198078695685, "clip_ratio/high_mean": 0.0005776039688498713, "clip_ratio/low_mean": 0.0012451171824068297, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018227211548946798, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 947.0, "completions/mean_length": 901.0250244140625, "completions/mean_terminated_length": 576.8181762695312, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.09495636075735092, "epoch": 0.16609783845278725, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.010349164716899395, "kl": 0.00632942549418658, "learning_rate": 7.510238907849829e-06, "loss": -0.0229, "num_tokens": 31309043.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 1.2201621532440186, "sampling/importance_sampling_ratio/mean": 0.28675585985183716, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.046659231185913, "sampling/sampling_logp_difference/mean": 0.010928082279860973, "step": 292, "step_time": 69.22555643832311 }, { "clip_ratio/high_max": 0.011675312111037783, "clip_ratio/high_mean": 0.004155867009103531, "clip_ratio/low_mean": 0.0011718750283762347, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005327741877408698, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 763.0750122070312, "completions/mean_terminated_length": 589.125, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.09784151241183281, "epoch": 0.16666666666666666, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026353465393185616, "kl": 0.00992821587715298, "learning_rate": 7.505119453924915e-06, "loss": 0.1279, "num_tokens": 31393734.0, "reward": 1.2000000476837158, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.3026199340820312, "sampling/importance_sampling_ratio/mean": 0.4419167637825012, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.19731068611145, "sampling/sampling_logp_difference/mean": 0.012207907624542713, "step": 293, "step_time": 67.7336155581288 }, { "clip_ratio/high_max": 0.011165014060679823, "clip_ratio/high_mean": 0.0033493937080493197, "clip_ratio/low_mean": 0.002001953136641532, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005351346917450428, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 963.0, "completions/mean_length": 882.875, "completions/mean_terminated_length": 647.6666870117188, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.1285398481413722, "epoch": 0.16723549488054607, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.012427456676959991, "kl": 0.009705217671580613, "learning_rate": 7.5e-06, "loss": 0.0185, "num_tokens": 31500801.0, "reward": 0.699999988079071, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.6996065378189087, "sampling/importance_sampling_ratio/mean": 0.3405228555202484, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2164466381073, "sampling/sampling_logp_difference/mean": 0.013838238082826138, "step": 294, "step_time": 75.25753933750093 }, { "clip_ratio/high_max": 0.011625725077465177, "clip_ratio/high_mean": 0.003920620831195265, "clip_ratio/low_mean": 0.0012939453226863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005214566102949902, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 908.0, "completions/mean_length": 782.4000244140625, "completions/mean_terminated_length": 584.727294921875, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.13686947710812092, "epoch": 0.16780432309442547, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.030156334862113, "kl": 0.010554489446803927, "learning_rate": 7.494880546075085e-06, "loss": -0.0056, "num_tokens": 31603729.0, "reward": 1.024999976158142, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.5709028244018555, "sampling/importance_sampling_ratio/mean": 0.5123332142829895, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6515446901321411, "sampling/sampling_logp_difference/mean": 0.015470114536583424, "step": 295, "step_time": 69.70924127218314 }, { "clip_ratio/high_max": 0.010745985433459282, "clip_ratio/high_mean": 0.0032053500035544857, "clip_ratio/low_mean": 0.00448371097445488, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.007689060992561281, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 958.7250366210938, "completions/mean_terminated_length": 837.5000610351562, "completions/min_length": 640.0, "completions/min_terminated_length": 640.0, "entropy": 0.15706693194806576, "epoch": 0.16837315130830488, "frac_reward_zero_std": 0.0, "grad_norm": 0.007892712019383907, "kl": 0.008182629011571407, "learning_rate": 7.489761092150171e-06, "loss": 0.0126, "num_tokens": 31715350.0, "reward": 0.6000000238418579, "reward_std": 0.8711913824081421, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 0.7279289364814758, "sampling/importance_sampling_ratio/mean": 0.14341934025287628, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7741328477859497, "sampling/sampling_logp_difference/mean": 0.015454255044460297, "step": 296, "step_time": 74.63869625935331 }, { "clip_ratio/high_max": 0.006464436009991914, "clip_ratio/high_mean": 0.0012928872019983828, "clip_ratio/low_mean": 0.0016845702957652975, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029774574213661253, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 986.875, "completions/mean_terminated_length": 776.5, "completions/min_length": 698.0, "completions/min_terminated_length": 698.0, "entropy": 0.08539230469614267, "epoch": 0.1689419795221843, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.021153323352336884, "kl": 0.00504691555397585, "learning_rate": 7.4846416382252565e-06, "loss": 0.0409, "num_tokens": 31822617.0, "reward": 0.2750000059604645, "reward_std": 0.6788942813873291, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.1663146018981934, "sampling/importance_sampling_ratio/mean": 0.5512695908546448, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4420169591903687, "sampling/sampling_logp_difference/mean": 0.0095754936337471, "step": 297, "step_time": 73.56742866616696 }, { "clip_ratio/high_max": 0.014421291532926261, "clip_ratio/high_mean": 0.008173734589945525, "clip_ratio/low_mean": 0.000972833797277417, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009146568365395069, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 694.6000366210938, "completions/mean_terminated_length": 636.4705810546875, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.1096894983202219, "epoch": 0.1695108077360637, "frac_reward_zero_std": 0.0, "grad_norm": 0.023499716073274612, "kl": 0.008429303939919919, "learning_rate": 7.479522184300342e-06, "loss": 0.0577, "num_tokens": 31922217.0, "reward": 1.625, "reward_std": 0.7741827964782715, "rewards/format_reward/mean": 0.824999988079071, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.800000011920929, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.891352415084839, "sampling/importance_sampling_ratio/mean": 0.4671901762485504, "sampling/importance_sampling_ratio/min": 0.002439268631860614, "sampling/sampling_logp_difference/max": 2.002566337585449, "sampling/sampling_logp_difference/mean": 0.013547536917030811, "step": 298, "step_time": 70.64878061483614 }, { "clip_ratio/high_max": 0.013571275980211794, "clip_ratio/high_mean": 0.0036783470131922513, "clip_ratio/low_mean": 0.0016960089124040678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0053743558819405735, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 966.0, "completions/mean_length": 818.4750366210938, "completions/mean_terminated_length": 632.5238037109375, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "entropy": 0.13062175642699003, "epoch": 0.17007963594994313, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01920648291707039, "kl": 0.010259098082315177, "learning_rate": 7.4744027303754265e-06, "loss": -0.0241, "num_tokens": 32031612.0, "reward": 0.9750000238418579, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.891176223754883, "sampling/importance_sampling_ratio/mean": 0.33836686611175537, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.699399709701538, "sampling/sampling_logp_difference/mean": 0.014948609285056591, "step": 299, "step_time": 74.54162193043157 }, { "clip_ratio/high_max": 0.014616990927606821, "clip_ratio/high_mean": 0.004417213931446895, "clip_ratio/low_mean": 0.00209960939901066, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006516823312267661, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 851.0, "completions/mean_length": 823.9750366210938, "completions/mean_terminated_length": 523.9375, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.13555280584841967, "epoch": 0.17064846416382254, "frac_reward_zero_std": 0.0, "grad_norm": 0.02777312695980072, "kl": 0.01902917167171836, "learning_rate": 7.469283276450512e-06, "loss": 0.0024, "num_tokens": 32136107.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.9135522842407227, "sampling/importance_sampling_ratio/mean": 0.38683947920799255, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.719226837158203, "sampling/sampling_logp_difference/mean": 0.014016680419445038, "step": 300, "step_time": 72.88446141500026 }, { "clip_ratio/high_max": 0.006272896454902366, "clip_ratio/high_mean": 0.0015306529166991822, "clip_ratio/low_mean": 0.0010009765610448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0025316294631920755, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 815.0, "completions/mean_length": 957.875, "completions/mean_terminated_length": 583.1666870117188, "completions/min_length": 451.0, "completions/min_terminated_length": 451.0, "entropy": 0.13821107614785433, "epoch": 0.17121729237770195, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.015698449686169624, "kl": 0.007816775352694094, "learning_rate": 7.464163822525597e-06, "loss": 0.0142, "num_tokens": 32251302.0, "reward": 0.30000001192092896, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.9220170974731445, "sampling/importance_sampling_ratio/mean": 0.46915093064308167, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9746476411819458, "sampling/sampling_logp_difference/mean": 0.014192732982337475, "step": 301, "step_time": 76.93911539996043 }, { "clip_ratio/high_max": 0.014203104074113071, "clip_ratio/high_mean": 0.004911713520414196, "clip_ratio/low_mean": 0.002477190369972959, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007388903992250562, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 860.6749877929688, "completions/mean_terminated_length": 661.0555419921875, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.1394074484705925, "epoch": 0.17178612059158135, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.016616739332675934, "kl": 0.010458960547111928, "learning_rate": 7.459044368600683e-06, "loss": -0.0738, "num_tokens": 32358737.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.2682207822799683, "sampling/importance_sampling_ratio/mean": 0.2607055902481079, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0745108127593994, "sampling/sampling_logp_difference/mean": 0.014704093337059021, "step": 302, "step_time": 76.23871824517846 }, { "clip_ratio/high_max": 0.015722445910796523, "clip_ratio/high_mean": 0.004236991459038109, "clip_ratio/low_mean": 0.0013671875203726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005604178819339722, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 802.9500122070312, "completions/mean_terminated_length": 581.9000244140625, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.11111375875771046, "epoch": 0.17235494880546076, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02930750697851181, "kl": 0.00808851100737229, "learning_rate": 7.453924914675767e-06, "loss": 0.0188, "num_tokens": 32466047.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.843416452407837, "sampling/importance_sampling_ratio/mean": 0.5453240871429443, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4195024967193604, "sampling/sampling_logp_difference/mean": 0.013069103471934795, "step": 303, "step_time": 73.76987687824294 }, { "clip_ratio/high_max": 0.009331541310530156, "clip_ratio/high_mean": 0.0023633930541109294, "clip_ratio/low_mean": 0.0012908562148368219, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003654249245300889, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 725.4750366210938, "completions/mean_terminated_length": 564.7307739257812, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.12293118890374899, "epoch": 0.17292377701934017, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01669652760028839, "kl": 0.007497403712477535, "learning_rate": 7.448805460750854e-06, "loss": 0.0327, "num_tokens": 32570266.0, "reward": 0.9000000357627869, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.6462393999099731, "sampling/importance_sampling_ratio/mean": 0.4066033363342285, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6783726215362549, "sampling/sampling_logp_difference/mean": 0.014206293039023876, "step": 304, "step_time": 70.49410978192464 }, { "clip_ratio/high_max": 0.014528184197843075, "clip_ratio/high_mean": 0.006304037990048528, "clip_ratio/low_mean": 0.0016136537960846908, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007917691837064922, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 743.4750366210938, "completions/mean_terminated_length": 623.25, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.12376177590340376, "epoch": 0.17349260523321958, "frac_reward_zero_std": 0.0, "grad_norm": 0.031208964064717293, "kl": 0.009135073516517878, "learning_rate": 7.443686006825939e-06, "loss": -0.011, "num_tokens": 32665941.0, "reward": 1.1750000715255737, "reward_std": 0.873762845993042, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.1060614585876465, "sampling/importance_sampling_ratio/mean": 0.47369447350502014, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0261404514312744, "sampling/sampling_logp_difference/mean": 0.014206781052052975, "step": 305, "step_time": 72.08539398643188 }, { "clip_ratio/high_max": 0.00963384669739753, "clip_ratio/high_mean": 0.002343114872928709, "clip_ratio/low_mean": 0.0011474609418655746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003490575822070241, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 961.0, "completions/mean_terminated_length": 664.0, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.08595345541834831, "epoch": 0.17406143344709898, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.012202218174934387, "kl": 0.00811865576542914, "learning_rate": 7.4385665529010246e-06, "loss": 0.0566, "num_tokens": 32779765.0, "reward": 0.3499999940395355, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.8207294940948486, "sampling/importance_sampling_ratio/mean": 0.40689414739608765, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5996193885803223, "sampling/sampling_logp_difference/mean": 0.009607695043087006, "step": 306, "step_time": 75.82873610826209 }, { "clip_ratio/high_max": 0.009761171881109476, "clip_ratio/high_mean": 0.002776284789433703, "clip_ratio/low_mean": 0.001342773446594947, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004119058132346254, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 910.5499877929688, "completions/mean_terminated_length": 456.75, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "entropy": 0.0916958013549447, "epoch": 0.1746302616609784, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02786877565085888, "kl": 0.00664434878854081, "learning_rate": 7.433447098976109e-06, "loss": 0.1257, "num_tokens": 32891083.0, "reward": 0.375, "reward_std": 0.7741827964782715, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.9988739490509033, "sampling/importance_sampling_ratio/mean": 0.5132132768630981, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9904565811157227, "sampling/sampling_logp_difference/mean": 0.010604417882859707, "step": 307, "step_time": 74.88182149012573 }, { "clip_ratio/high_max": 0.01679002831224352, "clip_ratio/high_mean": 0.005584540456766263, "clip_ratio/low_mean": 0.0019531250327418093, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00753766531124711, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 901.2750244140625, "completions/mean_terminated_length": 717.1875, "completions/min_length": 474.0, "completions/min_terminated_length": 474.0, "entropy": 0.16781958006322384, "epoch": 0.1751990898748578, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018984535709023476, "kl": 0.00938257429515943, "learning_rate": 7.4283276450511946e-06, "loss": -0.03, "num_tokens": 33001486.0, "reward": 0.675000011920929, "reward_std": 0.888314425945282, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.5634793043136597, "sampling/importance_sampling_ratio/mean": 0.24382257461547852, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9478964805603027, "sampling/sampling_logp_difference/mean": 0.01672477088868618, "step": 308, "step_time": 75.21216008183546 }, { "clip_ratio/high_max": 0.010289058162015863, "clip_ratio/high_mean": 0.0033278949304076377, "clip_ratio/low_mean": 0.0013671875203726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00469508244714234, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 943.0, "completions/mean_length": 903.9500122070312, "completions/mean_terminated_length": 654.6154174804688, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "entropy": 0.11812427267432213, "epoch": 0.1757679180887372, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.026269767433404922, "kl": 0.015907681896351278, "learning_rate": 7.42320819112628e-06, "loss": -0.0071, "num_tokens": 33111940.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.3688547611236572, "sampling/importance_sampling_ratio/mean": 0.4215361475944519, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7579660415649414, "sampling/sampling_logp_difference/mean": 0.012141323648393154, "step": 309, "step_time": 75.06511952634901 }, { "clip_ratio/high_max": 0.009994600201025605, "clip_ratio/high_mean": 0.0023255923588294536, "clip_ratio/low_mean": 0.0013427734447759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003668365861813072, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 850.7000122070312, "completions/mean_terminated_length": 561.86669921875, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.12336515076458454, "epoch": 0.17633674630261661, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.020916465669870377, "kl": 0.008280242327600718, "learning_rate": 7.4180887372013654e-06, "loss": 0.0498, "num_tokens": 33215688.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.8806220293045044, "sampling/importance_sampling_ratio/mean": 0.46984782814979553, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7144343852996826, "sampling/sampling_logp_difference/mean": 0.01367504894733429, "step": 310, "step_time": 73.53580675739795 }, { "clip_ratio/high_max": 0.009990662627387792, "clip_ratio/high_mean": 0.0028087528771720827, "clip_ratio/low_mean": 0.003686523501528427, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.006495276436908171, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 949.4000244140625, "completions/mean_terminated_length": 725.6000366210938, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "entropy": 0.14687662106007338, "epoch": 0.17690557451649602, "frac_reward_zero_std": 0.0, "grad_norm": 0.04913950711488724, "kl": 0.015571375493891537, "learning_rate": 7.41296928327645e-06, "loss": 0.0101, "num_tokens": 33328640.0, "reward": 0.45000001788139343, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.7512333393096924, "sampling/importance_sampling_ratio/mean": 0.37682995200157166, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.950289249420166, "sampling/sampling_logp_difference/mean": 0.015004533343017101, "step": 311, "step_time": 75.65223229001276 }, { "clip_ratio/high_max": 0.012890788610093296, "clip_ratio/high_mean": 0.004137244133744389, "clip_ratio/low_mean": 0.0021728515603172127, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006310095835942775, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 892.4000244140625, "completions/mean_terminated_length": 648.0, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.18805759213864803, "epoch": 0.17747440273037543, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02301211655139923, "kl": 0.008897600637283176, "learning_rate": 7.407849829351536e-06, "loss": -0.007, "num_tokens": 33443632.0, "reward": 0.550000011920929, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.2914469242095947, "sampling/importance_sampling_ratio/mean": 0.4240245819091797, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0388262271881104, "sampling/sampling_logp_difference/mean": 0.016701476648449898, "step": 312, "step_time": 78.8995828141924 }, { "clip_ratio/high_max": 0.011888856155565009, "clip_ratio/high_mean": 0.0033567799473530613, "clip_ratio/low_mean": 0.001944387306139106, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005301167228026316, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 660.75, "completions/mean_terminated_length": 555.290283203125, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.1505036409944296, "epoch": 0.17804323094425484, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04120967909693718, "kl": 0.012986192188691348, "learning_rate": 7.402730375426622e-06, "loss": 0.0159, "num_tokens": 33544782.0, "reward": 1.350000023841858, "reward_std": 0.8335896730422974, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.8393681049346924, "sampling/importance_sampling_ratio/mean": 0.7040973901748657, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3118386268615723, "sampling/sampling_logp_difference/mean": 0.016760677099227905, "step": 313, "step_time": 68.1271216548048 }, { "clip_ratio/high_max": 0.013029511901549995, "clip_ratio/high_mean": 0.003895821573678404, "clip_ratio/low_mean": 0.0037841796583961695, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007680001232074574, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 939.2750244140625, "completions/mean_terminated_length": 715.9091186523438, "completions/min_length": 452.0, "completions/min_terminated_length": 452.0, "entropy": 0.18463767413049936, "epoch": 0.17861205915813425, "frac_reward_zero_std": 0.0, "grad_norm": 0.014193271286785603, "kl": 0.008511510503012687, "learning_rate": 7.397610921501707e-06, "loss": 0.0779, "num_tokens": 33658353.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.018855571746826, "sampling/importance_sampling_ratio/mean": 0.275046169757843, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8970026969909668, "sampling/sampling_logp_difference/mean": 0.01735839806497097, "step": 314, "step_time": 75.71505781216547 }, { "clip_ratio/high_max": 0.018006160273216665, "clip_ratio/high_mean": 0.006708045373670757, "clip_ratio/low_mean": 0.0022819892619736493, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008990034635644406, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 778.0, "completions/mean_length": 789.7000122070312, "completions/mean_terminated_length": 530.73681640625, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.155140683054924, "epoch": 0.17918088737201365, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01693115569651127, "kl": 0.010141855804249644, "learning_rate": 7.392491467576792e-06, "loss": 0.0426, "num_tokens": 33767357.0, "reward": 0.824999988079071, "reward_std": 0.9306049942970276, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.509744644165039, "sampling/importance_sampling_ratio/mean": 0.3664425313472748, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4725663661956787, "sampling/sampling_logp_difference/mean": 0.015957802534103394, "step": 315, "step_time": 74.09035351290368 }, { "clip_ratio/high_max": 0.013184624840505421, "clip_ratio/high_mean": 0.0034316743694944307, "clip_ratio/low_mean": 0.0006347656362777343, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004066440014867112, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 832.5250244140625, "completions/mean_terminated_length": 598.5, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.09633103851228952, "epoch": 0.17974971558589306, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.02584843896329403, "kl": 0.008602131041698158, "learning_rate": 7.387372013651877e-06, "loss": -0.0496, "num_tokens": 33865826.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.900768518447876, "sampling/importance_sampling_ratio/mean": 0.4664815068244934, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.355722427368164, "sampling/sampling_logp_difference/mean": 0.011512191966176033, "step": 316, "step_time": 74.10110257891938 }, { "clip_ratio/high_max": 0.002786612487398088, "clip_ratio/high_mean": 0.000557322520762682, "clip_ratio/low_mean": 0.00043945313336735126, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009967756541300332, "completions/clipped_ratio": 0.949999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 961.0, "completions/mean_length": 1013.6500244140625, "completions/mean_terminated_length": 817.0, "completions/min_length": 673.0, "completions/min_terminated_length": 673.0, "entropy": 0.04489988787099719, "epoch": 0.18031854379977247, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.01098895724862814, "kl": 0.007048865809338167, "learning_rate": 7.382252559726963e-06, "loss": 0.0435, "num_tokens": 33968588.0, "reward": 0.10000000149011612, "reward_std": 0.4414428472518921, "rewards/format_reward/mean": 0.05000000074505806, "rewards/format_reward/std": 0.22072142362594604, "rewards/gpt_judge_reward/mean": 0.05000000074505806, "rewards/gpt_judge_reward/std": 0.22072142362594604, "sampling/importance_sampling_ratio/max": 2.1347410678863525, "sampling/importance_sampling_ratio/mean": 0.5140087008476257, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.562093257904053, "sampling/sampling_logp_difference/mean": 0.006766286678612232, "step": 317, "step_time": 72.22716991393827 }, { "clip_ratio/high_max": 0.009362563752802089, "clip_ratio/high_mean": 0.0031006076897028834, "clip_ratio/low_mean": 0.0025393969262950122, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.005640004499582574, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 907.0, "completions/mean_length": 859.5, "completions/mean_terminated_length": 517.84619140625, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.07704338803887367, "epoch": 0.18088737201365188, "frac_reward_zero_std": 0.0, "grad_norm": 0.03309468552470207, "kl": 0.011115649773273617, "learning_rate": 7.377133105802047e-06, "loss": -0.0292, "num_tokens": 34076816.0, "reward": 0.625, "reward_std": 0.9250779151916504, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409550309181213, "sampling/importance_sampling_ratio/max": 2.838895559310913, "sampling/importance_sampling_ratio/mean": 0.36010318994522095, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.905025005340576, "sampling/sampling_logp_difference/mean": 0.010435118339955807, "step": 318, "step_time": 72.90802076342516 }, { "clip_ratio/high_max": 0.01894579629879445, "clip_ratio/high_mean": 0.005934771295869723, "clip_ratio/low_mean": 0.0007140544512367342, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006648825772572309, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 962.0, "completions/mean_length": 812.9750366210938, "completions/mean_terminated_length": 496.4375, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.09082319959998131, "epoch": 0.18145620022753128, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03949662297964096, "kl": 0.017228885437361896, "learning_rate": 7.372013651877133e-06, "loss": -0.0053, "num_tokens": 34181367.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.7183797359466553, "sampling/importance_sampling_ratio/mean": 0.6087765693664551, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.232668399810791, "sampling/sampling_logp_difference/mean": 0.011870686896145344, "step": 319, "step_time": 70.72719837375917 }, { "clip_ratio/high_max": 0.01335970827494748, "clip_ratio/high_mean": 0.0029717017787334044, "clip_ratio/low_mean": 0.0013671875349245965, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004338889237260446, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 969.0, "completions/mean_length": 807.0250244140625, "completions/mean_terminated_length": 513.4705810546875, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.09150080475956202, "epoch": 0.1820250284414107, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02296055667102337, "kl": 0.006960459577385336, "learning_rate": 7.366894197952219e-06, "loss": 0.0468, "num_tokens": 34290144.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.1601037979125977, "sampling/importance_sampling_ratio/mean": 0.4263160824775696, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.074500560760498, "sampling/sampling_logp_difference/mean": 0.010591665282845497, "step": 320, "step_time": 74.29590758285485 }, { "clip_ratio/high_max": 0.012313450162764639, "clip_ratio/high_mean": 0.00382894478389062, "clip_ratio/low_mean": 0.0025401839811820537, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0063691287650726736, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 876.6000366210938, "completions/mean_terminated_length": 655.5, "completions/min_length": 394.0, "completions/min_terminated_length": 394.0, "entropy": 0.08650470664724708, "epoch": 0.1825938566552901, "frac_reward_zero_std": 0.0, "grad_norm": 0.027943801134824753, "kl": 0.0070888742338865995, "learning_rate": 7.361774744027304e-06, "loss": 0.1761, "num_tokens": 34396032.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.746549129486084, "sampling/importance_sampling_ratio/mean": 0.661094069480896, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.428415298461914, "sampling/sampling_logp_difference/mean": 0.010408038273453712, "step": 321, "step_time": 73.5366215549875 }, { "clip_ratio/high_max": 0.014132917858660221, "clip_ratio/high_mean": 0.00457284229923971, "clip_ratio/low_mean": 0.0028854486881755292, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007458290841896087, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.0, "completions/mean_length": 881.4500122070312, "completions/mean_terminated_length": 738.9000244140625, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.14314919244498014, "epoch": 0.1831626848691695, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0635308101773262, "kl": 0.009571289992891252, "learning_rate": 7.356655290102389e-06, "loss": -0.0573, "num_tokens": 34499970.0, "reward": 0.824999988079071, "reward_std": 0.9306049942970276, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.89424467086792, "sampling/importance_sampling_ratio/mean": 0.5760738849639893, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.331427812576294, "sampling/sampling_logp_difference/mean": 0.014777674339711666, "step": 322, "step_time": 73.78493907838129 }, { "clip_ratio/high_max": 0.013891906768549234, "clip_ratio/high_mean": 0.0038410607812693343, "clip_ratio/low_mean": 0.0015869140625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005427974800113589, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 783.7000122070312, "completions/mean_terminated_length": 543.4000244140625, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.127285273745656, "epoch": 0.18373151308304891, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0532396137714386, "kl": 0.00955429021269083, "learning_rate": 7.351535836177474e-06, "loss": 0.0742, "num_tokens": 34610854.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.832563877105713, "sampling/importance_sampling_ratio/mean": 0.5914608240127563, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9588357210159302, "sampling/sampling_logp_difference/mean": 0.01362581830471754, "step": 323, "step_time": 74.85064996499568 }, { "clip_ratio/high_max": 0.012481503479648381, "clip_ratio/high_mean": 0.0032958043448161334, "clip_ratio/low_mean": 0.0020138648760621436, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00530966927180998, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 898.125, "completions/mean_terminated_length": 709.3125, "completions/min_length": 449.0, "completions/min_terminated_length": 449.0, "entropy": 0.13326191436499357, "epoch": 0.18430034129692832, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01122372318059206, "kl": 0.008832281164359301, "learning_rate": 7.34641638225256e-06, "loss": -0.0083, "num_tokens": 34718739.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.4443349838256836, "sampling/importance_sampling_ratio/mean": 0.32032763957977295, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2294530868530273, "sampling/sampling_logp_difference/mean": 0.013792961835861206, "step": 324, "step_time": 72.86919420957565 }, { "clip_ratio/high_max": 0.00945161544950679, "clip_ratio/high_mean": 0.002540095170843415, "clip_ratio/low_mean": 0.0008056640763243195, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033457592071499676, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 887.0, "completions/mean_length": 907.9500122070312, "completions/mean_terminated_length": 637.1666870117188, "completions/min_length": 475.0, "completions/min_terminated_length": 475.0, "entropy": 0.12711833510547876, "epoch": 0.18486916951080773, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015288025140762329, "kl": 0.006960259866900742, "learning_rate": 7.341296928327645e-06, "loss": -0.0203, "num_tokens": 34833865.0, "reward": 0.6000000238418579, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.5674524307250977, "sampling/importance_sampling_ratio/mean": 0.23599085211753845, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.102126359939575, "sampling/sampling_logp_difference/mean": 0.012141003273427486, "step": 325, "step_time": 76.25598098873161 }, { "clip_ratio/high_max": 0.014128530048765242, "clip_ratio/high_mean": 0.00407574133714661, "clip_ratio/low_mean": 0.002612304699141532, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006688046152703464, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 890.0, "completions/mean_length": 892.2250366210938, "completions/mean_terminated_length": 544.8181762695312, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.12030760757625103, "epoch": 0.18543799772468714, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04598119482398033, "kl": 0.009182940644677728, "learning_rate": 7.33617747440273e-06, "loss": -0.1358, "num_tokens": 34943346.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.8238391876220703, "sampling/importance_sampling_ratio/mean": 0.5241624712944031, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5551631450653076, "sampling/sampling_logp_difference/mean": 0.012720832601189613, "step": 326, "step_time": 73.24214843730442 }, { "clip_ratio/high_max": 0.012146060762461275, "clip_ratio/high_mean": 0.003239443110942375, "clip_ratio/low_mean": 0.0013714448505197652, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00461088793235831, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 967.0, "completions/mean_length": 888.2999877929688, "completions/mean_terminated_length": 662.1333618164062, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.1535190101712942, "epoch": 0.18600682593856654, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.017291231080889702, "kl": 0.007937664806377143, "learning_rate": 7.331058020477816e-06, "loss": 0.0594, "num_tokens": 35056542.0, "reward": 0.675000011920929, "reward_std": 0.9167249798774719, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.182201385498047, "sampling/importance_sampling_ratio/mean": 0.36396998167037964, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9508609771728516, "sampling/sampling_logp_difference/mean": 0.01501341164112091, "step": 327, "step_time": 75.65809159702621 }, { "clip_ratio/high_max": 0.010542816889937967, "clip_ratio/high_mean": 0.003030981642950792, "clip_ratio/low_mean": 0.0019042968851863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0049352785281371325, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 928.4500122070312, "completions/mean_terminated_length": 705.5, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "entropy": 0.11944451462477446, "epoch": 0.18657565415244595, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.017697438597679138, "kl": 0.00903070456115529, "learning_rate": 7.3259385665529016e-06, "loss": -0.0174, "num_tokens": 35161328.0, "reward": 0.6000000238418579, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.4510884284973145, "sampling/importance_sampling_ratio/mean": 0.38384097814559937, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.659928798675537, "sampling/sampling_logp_difference/mean": 0.012396869249641895, "step": 328, "step_time": 77.02044058707543 }, { "clip_ratio/high_max": 0.01544576813466847, "clip_ratio/high_mean": 0.004503633797867224, "clip_ratio/low_mean": 0.0023437499949068297, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0068473839201033115, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 930.0, "completions/mean_length": 873.6749877929688, "completions/mean_terminated_length": 594.5, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.16298873722553253, "epoch": 0.18714448236632536, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.035688694566488266, "kl": 0.011328321881592274, "learning_rate": 7.320819112627987e-06, "loss": -0.0397, "num_tokens": 35268195.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.5107369422912598, "sampling/importance_sampling_ratio/mean": 0.38934728503227234, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5538923740386963, "sampling/sampling_logp_difference/mean": 0.016994785517454147, "step": 329, "step_time": 73.88441499811597 }, { "clip_ratio/high_max": 0.018941048299893737, "clip_ratio/high_mean": 0.005607474144198932, "clip_ratio/low_mean": 0.0017578125189174898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00736528669949621, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 880.7750244140625, "completions/mean_terminated_length": 614.7857666015625, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.1304479669779539, "epoch": 0.18771331058020477, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022617587819695473, "kl": 0.019714852096512914, "learning_rate": 7.315699658703072e-06, "loss": 0.009, "num_tokens": 35372954.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.8020179271698, "sampling/importance_sampling_ratio/mean": 0.31168311834335327, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.5538105964660645, "sampling/sampling_logp_difference/mean": 0.014186284504830837, "step": 330, "step_time": 74.7507617129013 }, { "clip_ratio/high_max": 0.009171261393930763, "clip_ratio/high_mean": 0.0019760962313739583, "clip_ratio/low_mean": 0.0014467389482888393, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034228351287310943, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 739.3500366210938, "completions/mean_terminated_length": 549.5833740234375, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.10100615210831165, "epoch": 0.18828213879408418, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.0304879080504179, "kl": 0.008512251486536115, "learning_rate": 7.310580204778157e-06, "loss": 0.0005, "num_tokens": 35469696.0, "reward": 0.9750000238418579, "reward_std": 0.8911962509155273, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.1127614974975586, "sampling/importance_sampling_ratio/mean": 0.4305863380432129, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2338449954986572, "sampling/sampling_logp_difference/mean": 0.01281294971704483, "step": 331, "step_time": 72.86206073197536 }, { "clip_ratio/high_max": 0.017518381937406957, "clip_ratio/high_mean": 0.0051712741842493415, "clip_ratio/low_mean": 0.003256619311287068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008427893568295985, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 809.9500122070312, "completions/mean_terminated_length": 616.2857055664062, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.16818732023239136, "epoch": 0.18885096700796358, "frac_reward_zero_std": 0.0, "grad_norm": 0.06834868341684341, "kl": 0.027915907907299697, "learning_rate": 7.3054607508532424e-06, "loss": 0.0209, "num_tokens": 35581542.0, "reward": 0.824999988079071, "reward_std": 0.873762845993042, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.7120988368988037, "sampling/importance_sampling_ratio/mean": 0.5197417140007019, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7144336700439453, "sampling/sampling_logp_difference/mean": 0.017570732161402702, "step": 332, "step_time": 74.18639604304917 }, { "clip_ratio/high_max": 0.0083014662086498, "clip_ratio/high_mean": 0.0026649619794625323, "clip_ratio/low_mean": 0.0009033203168655746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003568282292690128, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 914.4249877929688, "completions/mean_terminated_length": 585.7000122070312, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.09755624085664749, "epoch": 0.189419795221843, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.012409510090947151, "kl": 0.011828033719211817, "learning_rate": 7.300341296928328e-06, "loss": -0.0319, "num_tokens": 35687703.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.7085736989974976, "sampling/importance_sampling_ratio/mean": 0.4296841025352478, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1361172199249268, "sampling/sampling_logp_difference/mean": 0.011852830648422241, "step": 333, "step_time": 72.63637417391874 }, { "clip_ratio/high_max": 0.011538611201103777, "clip_ratio/high_mean": 0.003801804588874802, "clip_ratio/low_mean": 0.0018066406082652975, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005608445411780849, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 803.1500244140625, "completions/mean_terminated_length": 559.0526123046875, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.08071208978071809, "epoch": 0.1899886234357224, "frac_reward_zero_std": 0.0, "grad_norm": 0.05340778827667236, "kl": 0.011806611786596477, "learning_rate": 7.2952218430034125e-06, "loss": -0.1013, "num_tokens": 35787261.0, "reward": 0.8500000238418579, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.873422384262085, "sampling/importance_sampling_ratio/mean": 0.42824459075927734, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.474449634552002, "sampling/sampling_logp_difference/mean": 0.011280317790806293, "step": 334, "step_time": 70.40723159792833 }, { "clip_ratio/high_max": 0.016989033319987357, "clip_ratio/high_mean": 0.00865827890811488, "clip_ratio/low_mean": 0.0024305933038704097, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.011088872328400612, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 780.9750366210938, "completions/mean_terminated_length": 650.1154174804688, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.17087940499186516, "epoch": 0.19055745164960183, "frac_reward_zero_std": 0.0, "grad_norm": 0.015620714984834194, "kl": 0.01777869346551597, "learning_rate": 7.290102389078499e-06, "loss": 0.0142, "num_tokens": 35888652.0, "reward": 1.2000000476837158, "reward_std": 0.9391758441925049, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.7883641719818115, "sampling/importance_sampling_ratio/mean": 0.2804202735424042, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.074207067489624, "sampling/sampling_logp_difference/mean": 0.01811692677438259, "step": 335, "step_time": 74.76608501398005 }, { "clip_ratio/high_max": 0.011408081511035562, "clip_ratio/high_mean": 0.0036898557154927403, "clip_ratio/low_mean": 0.0019881021726178005, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0056779579317662865, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 885.4500122070312, "completions/mean_terminated_length": 628.1428833007812, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.11323037184774876, "epoch": 0.19112627986348124, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04075677692890167, "kl": 0.012434731819666922, "learning_rate": 7.284982935153584e-06, "loss": 0.0129, "num_tokens": 35990510.0, "reward": 0.574999988079071, "reward_std": 0.8439072966575623, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.523267984390259, "sampling/importance_sampling_ratio/mean": 0.4156861901283264, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9445595741271973, "sampling/sampling_logp_difference/mean": 0.012890840880572796, "step": 336, "step_time": 70.93823801982217 }, { "clip_ratio/high_max": 0.016215192445088178, "clip_ratio/high_mean": 0.0046405808825511485, "clip_ratio/low_mean": 0.0038330077950377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00847358867758885, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 823.0, "completions/mean_length": 902.9249877929688, "completions/mean_terminated_length": 651.4615478515625, "completions/min_length": 465.0, "completions/min_terminated_length": 465.0, "entropy": 0.15523630194365978, "epoch": 0.19169510807736065, "frac_reward_zero_std": 0.0, "grad_norm": 0.0371466726064682, "kl": 0.009577609889674932, "learning_rate": 7.27986348122867e-06, "loss": -0.0768, "num_tokens": 36098323.0, "reward": 0.550000011920929, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 1.6356698274612427, "sampling/importance_sampling_ratio/mean": 0.341063916683197, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.730832576751709, "sampling/sampling_logp_difference/mean": 0.01583182066679001, "step": 337, "step_time": 72.0664025968872 }, { "clip_ratio/high_max": 0.009912071167491376, "clip_ratio/high_mean": 0.003150098433252424, "clip_ratio/low_mean": 0.0025109045891440473, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005661003058776259, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 874.75, "completions/mean_terminated_length": 526.5, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "entropy": 0.1018840977922082, "epoch": 0.19226393629124006, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03461890295147896, "kl": 0.009959797083865851, "learning_rate": 7.274744027303754e-06, "loss": -0.0763, "num_tokens": 36207905.0, "reward": 0.4749999940395355, "reward_std": 0.8161038756370544, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.365887403488159, "sampling/importance_sampling_ratio/mean": 0.3895576596260071, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.189159870147705, "sampling/sampling_logp_difference/mean": 0.01225382462143898, "step": 338, "step_time": 73.50189506239258 }, { "clip_ratio/high_max": 0.008169519365765154, "clip_ratio/high_mean": 0.0018481895822333172, "clip_ratio/low_mean": 0.0018066406519210432, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036548303032759577, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 943.0, "completions/mean_length": 773.2999877929688, "completions/mean_terminated_length": 588.0, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.08635669853538275, "epoch": 0.19283276450511946, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015022446401417255, "kl": 0.009138050547335297, "learning_rate": 7.26962457337884e-06, "loss": 0.0931, "num_tokens": 36316597.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.3884971141815186, "sampling/importance_sampling_ratio/mean": 0.34141144156455994, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8856909275054932, "sampling/sampling_logp_difference/mean": 0.012304154224693775, "step": 339, "step_time": 76.12402529222891 }, { "clip_ratio/high_max": 0.01905258945771493, "clip_ratio/high_mean": 0.005761696818808559, "clip_ratio/low_mean": 0.0009961069190467242, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006757803756045178, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 841.0, "completions/mean_length": 821.6000366210938, "completions/mean_terminated_length": 484.2666931152344, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.07622906100004911, "epoch": 0.19340159271899887, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 1.1132270097732544, "kl": 0.16759217088110745, "learning_rate": 7.264505119453925e-06, "loss": 0.0091, "num_tokens": 36412317.0, "reward": 0.699999988079071, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.988118886947632, "sampling/importance_sampling_ratio/mean": 0.37877726554870605, "sampling/importance_sampling_ratio/min": 5.614050113678104e-09, "sampling/sampling_logp_difference/max": 11.245932579040527, "sampling/sampling_logp_difference/mean": 0.012101531028747559, "step": 340, "step_time": 71.2048439651262 }, { "clip_ratio/high_max": 0.009988600970245898, "clip_ratio/high_mean": 0.003099909590673633, "clip_ratio/low_mean": 0.002795942418742925, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0058958520530723035, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 816.2750244140625, "completions/mean_terminated_length": 535.2352905273438, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.1513185678049922, "epoch": 0.19397042093287828, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013417430222034454, "kl": 0.012815676629543304, "learning_rate": 7.2593856655290105e-06, "loss": -0.0122, "num_tokens": 36519824.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.7858871221542358, "sampling/importance_sampling_ratio/mean": 0.22654493153095245, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3101625442504883, "sampling/sampling_logp_difference/mean": 0.01648935116827488, "step": 341, "step_time": 74.0302598096896 }, { "clip_ratio/high_max": 0.00947809498757124, "clip_ratio/high_mean": 0.002115560360834934, "clip_ratio/low_mean": 0.000732421884094947, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028479821994551457, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 798.9500122070312, "completions/mean_terminated_length": 423.86669921875, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.06384561583399773, "epoch": 0.1945392491467577, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.016299240291118622, "kl": 0.00920684786979109, "learning_rate": 7.254266211604095e-06, "loss": 0.0227, "num_tokens": 36623662.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.6213940382003784, "sampling/importance_sampling_ratio/mean": 0.4947964549064636, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9858413934707642, "sampling/sampling_logp_difference/mean": 0.009717556647956371, "step": 342, "step_time": 70.40367507422343 }, { "clip_ratio/high_max": 0.01121216465253383, "clip_ratio/high_mean": 0.0029545839643105865, "clip_ratio/low_mean": 0.003041819334612228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00599640334257856, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 939.4750366210938, "completions/mean_terminated_length": 763.923095703125, "completions/min_length": 452.0, "completions/min_terminated_length": 452.0, "entropy": 0.14686579816043377, "epoch": 0.1951080773606371, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023150045424699783, "kl": 0.009279794117901474, "learning_rate": 7.249146757679181e-06, "loss": -0.0706, "num_tokens": 36736377.0, "reward": 0.6000000238418579, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.9878015518188477, "sampling/importance_sampling_ratio/mean": 0.36594444513320923, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8851103782653809, "sampling/sampling_logp_difference/mean": 0.01534862257540226, "step": 343, "step_time": 76.69574335380457 }, { "clip_ratio/high_max": 0.008196966722607613, "clip_ratio/high_mean": 0.0018659794004634023, "clip_ratio/low_mean": 0.0017822265508584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036482059658737853, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 867.0750122070312, "completions/mean_terminated_length": 631.6875, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.08106834767386317, "epoch": 0.1956769055745165, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.008910397998988628, "kl": 0.011071101296693087, "learning_rate": 7.244027303754267e-06, "loss": 0.0124, "num_tokens": 36831572.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.576817035675049, "sampling/importance_sampling_ratio/mean": 0.336060106754303, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6571450233459473, "sampling/sampling_logp_difference/mean": 0.011607320047914982, "step": 344, "step_time": 73.54660332505591 }, { "clip_ratio/high_max": 0.008147630956955254, "clip_ratio/high_mean": 0.0018859076808439568, "clip_ratio/low_mean": 0.0010603856571833603, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029462932870956138, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.0, "completions/mean_length": 761.75, "completions/mean_terminated_length": 649.357177734375, "completions/min_length": 392.0, "completions/min_terminated_length": 392.0, "entropy": 0.13459716830402613, "epoch": 0.1962457337883959, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01780511997640133, "kl": 0.012642695102840662, "learning_rate": 7.238907849829352e-06, "loss": -0.0103, "num_tokens": 36936434.0, "reward": 1.3250000476837158, "reward_std": 0.9167249202728271, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.7465476989746094, "sampling/importance_sampling_ratio/mean": 0.48748669028282166, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.392490863800049, "sampling/sampling_logp_difference/mean": 0.01613587886095047, "step": 345, "step_time": 72.01267651375383 }, { "clip_ratio/high_max": 0.013314758311025798, "clip_ratio/high_mean": 0.005142503971001133, "clip_ratio/low_mean": 0.002490234386641532, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007632738386746496, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 880.2750244140625, "completions/mean_terminated_length": 721.4210815429688, "completions/min_length": 471.0, "completions/min_terminated_length": 471.0, "entropy": 0.13743538316339254, "epoch": 0.19681456200227532, "frac_reward_zero_std": 0.0, "grad_norm": 0.035070259124040604, "kl": 0.011157225002534688, "learning_rate": 7.233788395904437e-06, "loss": -0.0209, "num_tokens": 37049997.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.3886489868164062, "sampling/importance_sampling_ratio/mean": 0.46368274092674255, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0010986328125, "sampling/sampling_logp_difference/mean": 0.014916801825165749, "step": 346, "step_time": 76.42023315676488 }, { "clip_ratio/high_max": 0.013672226428752765, "clip_ratio/high_mean": 0.004789398190041538, "clip_ratio/low_mean": 0.002849447453627363, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00763884573825635, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 923.0, "completions/mean_length": 671.9750366210938, "completions/mean_terminated_length": 502.4814758300781, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.1279048603028059, "epoch": 0.19738339021615472, "frac_reward_zero_std": 0.0, "grad_norm": 0.04316962510347366, "kl": 0.014583053649403155, "learning_rate": 7.228668941979522e-06, "loss": 0.1104, "num_tokens": 37155212.0, "reward": 1.024999976158142, "reward_std": 0.8316649794578552, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.710969924926758, "sampling/importance_sampling_ratio/mean": 0.6398464441299438, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2121329307556152, "sampling/sampling_logp_difference/mean": 0.015925314277410507, "step": 347, "step_time": 73.73613278171979 }, { "clip_ratio/high_max": 0.014411676791496575, "clip_ratio/high_mean": 0.00313401322637219, "clip_ratio/low_mean": 0.0008300781355501385, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003964091301895678, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 574.0, "completions/mean_length": 919.25, "completions/mean_terminated_length": 425.4285888671875, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.06855457415804267, "epoch": 0.19795221843003413, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.013155738823115826, "kl": 0.007476285914890468, "learning_rate": 7.223549488054608e-06, "loss": 0.0179, "num_tokens": 37265726.0, "reward": 0.22500000894069672, "reward_std": 0.5304811596870422, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.05000000074505806, "rewards/gpt_judge_reward/std": 0.22072142362594604, "sampling/importance_sampling_ratio/max": 1.1193351745605469, "sampling/importance_sampling_ratio/mean": 0.35661056637763977, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.228789806365967, "sampling/sampling_logp_difference/mean": 0.008721678517758846, "step": 348, "step_time": 75.52126660081558 }, { "clip_ratio/high_max": 0.01709820213727653, "clip_ratio/high_mean": 0.006158852978842333, "clip_ratio/low_mean": 0.0026611327775754035, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008819985785521567, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 856.7999877929688, "completions/mean_terminated_length": 630.5882568359375, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.15899083483964205, "epoch": 0.19852104664391354, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022000033408403397, "kl": 0.012288979312870651, "learning_rate": 7.218430034129693e-06, "loss": 0.0479, "num_tokens": 37374894.0, "reward": 0.824999988079071, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.0992894172668457, "sampling/importance_sampling_ratio/mean": 0.32967036962509155, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5947656631469727, "sampling/sampling_logp_difference/mean": 0.017286214977502823, "step": 349, "step_time": 74.54389480222017 }, { "clip_ratio/high_max": 0.004677429562434554, "clip_ratio/high_mean": 0.0009354858921142295, "clip_ratio/low_mean": 0.0005371093793655746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001472595271479804, "completions/clipped_ratio": 0.925000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 943.0, "completions/mean_length": 1009.0750122070312, "completions/mean_terminated_length": 825.0, "completions/min_length": 726.0, "completions/min_terminated_length": 726.0, "entropy": 0.17946936562657356, "epoch": 0.19908987485779295, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.002470432547852397, "kl": 0.010099368286319077, "learning_rate": 7.213310580204778e-06, "loss": 0.0071, "num_tokens": 37489553.0, "reward": 0.10000000149011612, "reward_std": 0.3789323568344116, "rewards/format_reward/mean": 0.07500000298023224, "rewards/format_reward/std": 0.26674678921699524, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.26332688331604, "sampling/importance_sampling_ratio/mean": 0.3337005078792572, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.136477470397949, "sampling/sampling_logp_difference/mean": 0.016830559819936752, "step": 350, "step_time": 74.60748713486828 }, { "clip_ratio/high_max": 0.017975711380131543, "clip_ratio/high_mean": 0.0055994246795307845, "clip_ratio/low_mean": 0.0030761718517169356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008675596676766872, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 888.5250244140625, "completions/mean_terminated_length": 685.3125, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.14120159018784761, "epoch": 0.19965870307167236, "frac_reward_zero_std": 0.0, "grad_norm": 0.028743447735905647, "kl": 0.010530292754992843, "learning_rate": 7.208191126279864e-06, "loss": -0.0002, "num_tokens": 37600902.0, "reward": 0.7250000238418579, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.5316739082336426, "sampling/importance_sampling_ratio/mean": 0.307150661945343, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.8912222385406494, "sampling/sampling_logp_difference/mean": 0.015067226253449917, "step": 351, "step_time": 75.28946779877879 }, { "clip_ratio/high_max": 0.015461873961612582, "clip_ratio/high_mean": 0.005736349863582291, "clip_ratio/low_mean": 0.0016199629098991863, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007356312853517011, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 949.0, "completions/mean_length": 683.1000366210938, "completions/mean_terminated_length": 518.9629516601562, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.11863705422729254, "epoch": 0.20022753128555176, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026372388005256653, "kl": 0.014755059615708888, "learning_rate": 7.2030716723549494e-06, "loss": 0.0801, "num_tokens": 37704170.0, "reward": 1.274999976158142, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 1.7375081777572632, "sampling/importance_sampling_ratio/mean": 0.45294174551963806, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.270673990249634, "sampling/sampling_logp_difference/mean": 0.01524840947240591, "step": 352, "step_time": 74.0274592791684 }, { "clip_ratio/high_max": 0.01454114681109786, "clip_ratio/high_mean": 0.0035133206692989916, "clip_ratio/low_mean": 0.0025204276316799223, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006033748300978914, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 946.0, "completions/mean_terminated_length": 764.0, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.12599541246891022, "epoch": 0.20079635949943117, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024066930636763573, "kl": 0.009110438462812454, "learning_rate": 7.197952218430034e-06, "loss": -0.0475, "num_tokens": 37811850.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.1600165367126465, "sampling/importance_sampling_ratio/mean": 0.42584067583084106, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5627317428588867, "sampling/sampling_logp_difference/mean": 0.01398727297782898, "step": 353, "step_time": 77.2934970909264 }, { "clip_ratio/high_max": 0.011008941452018917, "clip_ratio/high_mean": 0.003988286087405868, "clip_ratio/low_mean": 0.002514648425858468, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.0065029344987124205, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 977.0, "completions/mean_length": 838.6000366210938, "completions/mean_terminated_length": 612.0, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.12028743512928486, "epoch": 0.20136518771331058, "frac_reward_zero_std": 0.0, "grad_norm": 0.02261364832520485, "kl": 0.011680688068736345, "learning_rate": 7.1928327645051195e-06, "loss": -0.0356, "num_tokens": 37921858.0, "reward": 0.8500000238418579, "reward_std": 0.9753369092941284, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.178084135055542, "sampling/importance_sampling_ratio/mean": 0.3721560835838318, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8834320306777954, "sampling/sampling_logp_difference/mean": 0.013839269988238811, "step": 354, "step_time": 75.65234799007885 }, { "clip_ratio/high_max": 0.0016592920292168856, "clip_ratio/high_mean": 0.0003318584058433771, "clip_ratio/low_mean": 0.0007568359396827873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010886943455261644, "completions/clipped_ratio": 0.9750000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 678.0, "completions/mean_length": 1015.3500366210938, "completions/mean_terminated_length": 678.0, "completions/min_length": 678.0, "completions/min_terminated_length": 678.0, "entropy": 0.061780947260558605, "epoch": 0.20193401592718999, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.016974668949842453, "kl": 0.00651052501052618, "learning_rate": 7.187713310580205e-06, "loss": -0.0275, "num_tokens": 38027808.0, "reward": 0.05000000074505806, "reward_std": 0.3162277638912201, "rewards/format_reward/mean": 0.02500000037252903, "rewards/format_reward/std": 0.15811389684677124, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 1.8019347190856934, "sampling/importance_sampling_ratio/mean": 0.38324156403541565, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.103673219680786, "sampling/sampling_logp_difference/mean": 0.008247840218245983, "step": 355, "step_time": 71.6920752802398 }, { "clip_ratio/high_max": 0.0062238272512331605, "clip_ratio/high_mean": 0.001751039148075506, "clip_ratio/low_mean": 0.0033447265886934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005095765693113208, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 979.2750244140625, "completions/mean_terminated_length": 725.8333740234375, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.12419057078659534, "epoch": 0.2025028441410694, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.011196800507605076, "kl": 0.009822159016039222, "learning_rate": 7.18259385665529e-06, "loss": 0.0366, "num_tokens": 38131995.0, "reward": 0.30000001192092896, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.1124966144561768, "sampling/importance_sampling_ratio/mean": 0.2779812216758728, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.951427936553955, "sampling/sampling_logp_difference/mean": 0.01363625843077898, "step": 356, "step_time": 72.67441144143231 }, { "clip_ratio/high_max": 0.01626583666075021, "clip_ratio/high_mean": 0.004626073568942957, "clip_ratio/low_mean": 0.0009277343961002771, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005553807888645679, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 862.0, "completions/mean_length": 783.7750244140625, "completions/mean_terminated_length": 566.4285888671875, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.09748072922229767, "epoch": 0.2030716723549488, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02003966085612774, "kl": 0.01237275858875364, "learning_rate": 7.177474402730375e-06, "loss": -0.0189, "num_tokens": 38237194.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.810357689857483, "sampling/importance_sampling_ratio/mean": 0.34961408376693726, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.18228816986084, "sampling/sampling_logp_difference/mean": 0.01289279293268919, "step": 357, "step_time": 71.41886187950149 }, { "clip_ratio/high_max": 0.00975035224109888, "clip_ratio/high_mean": 0.0024673220032127574, "clip_ratio/low_mean": 0.003222656319849193, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005689978221198544, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 806.0, "completions/mean_length": 929.9249877929688, "completions/mean_terminated_length": 486.4285888671875, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.11628114059567451, "epoch": 0.2036405005688282, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027988189831376076, "kl": 0.012366874259896576, "learning_rate": 7.17235494880546e-06, "loss": -0.0318, "num_tokens": 38342775.0, "reward": 0.32500001788139343, "reward_std": 0.7298577427864075, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 1.9242180585861206, "sampling/importance_sampling_ratio/mean": 0.32186639308929443, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.931506633758545, "sampling/sampling_logp_difference/mean": 0.013197523541748524, "step": 358, "step_time": 75.0681836407166 }, { "clip_ratio/high_max": 0.013616825570352376, "clip_ratio/high_mean": 0.004070541530381888, "clip_ratio/low_mean": 0.0011230468917347025, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0051935882365796715, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 900.0, "completions/mean_length": 835.625, "completions/mean_terminated_length": 580.7647094726562, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "entropy": 0.06302532879635692, "epoch": 0.20420932878270762, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026575040072202682, "kl": 0.009691912156995386, "learning_rate": 7.167235494880547e-06, "loss": -0.0581, "num_tokens": 38451248.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.830122709274292, "sampling/importance_sampling_ratio/mean": 0.5321499705314636, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0021979808807373, "sampling/sampling_logp_difference/mean": 0.009359843097627163, "step": 359, "step_time": 76.55596185149625 }, { "clip_ratio/high_max": 0.012177316239103675, "clip_ratio/high_mean": 0.0030960133590269834, "clip_ratio/low_mean": 0.0018564105412224308, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004952423914801329, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 838.8250122070312, "completions/mean_terminated_length": 612.5, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.1463069748133421, "epoch": 0.20477815699658702, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.035683371126651764, "kl": 0.012639354798011482, "learning_rate": 7.162116040955632e-06, "loss": -0.0728, "num_tokens": 38561985.0, "reward": 0.675000011920929, "reward_std": 0.8285761475563049, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.0718929767608643, "sampling/importance_sampling_ratio/mean": 0.5331700444221497, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.192206621170044, "sampling/sampling_logp_difference/mean": 0.015581324696540833, "step": 360, "step_time": 75.2950539807789 }, { "clip_ratio/high_max": 0.011380488285794854, "clip_ratio/high_mean": 0.003335904359119013, "clip_ratio/low_mean": 0.0006347656326397555, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0039706700190436095, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 828.0499877929688, "completions/mean_terminated_length": 534.125, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.09582998137921095, "epoch": 0.20534698521046643, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.16485516726970673, "kl": 0.025987807079218328, "learning_rate": 7.156996587030717e-06, "loss": 0.0552, "num_tokens": 38654291.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.9529149532318115, "sampling/importance_sampling_ratio/mean": 0.5041157007217407, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3422255516052246, "sampling/sampling_logp_difference/mean": 0.011328203603625298, "step": 361, "step_time": 72.93006210238673 }, { "clip_ratio/high_max": 0.012648408359382302, "clip_ratio/high_mean": 0.00381234084488824, "clip_ratio/low_mean": 0.0017578125116415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0055701535602565855, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 926.0, "completions/mean_length": 769.125, "completions/mean_terminated_length": 560.5909423828125, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.10919851157814264, "epoch": 0.20591581342434584, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.025447005406022072, "kl": 0.015180606045760214, "learning_rate": 7.151877133105802e-06, "loss": -0.0864, "num_tokens": 38750536.0, "reward": 1.100000023841858, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.9205820560455322, "sampling/importance_sampling_ratio/mean": 0.3614144027233124, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.578531265258789, "sampling/sampling_logp_difference/mean": 0.014083333313465118, "step": 362, "step_time": 69.20606104144827 }, { "clip_ratio/high_max": 0.016420223575551063, "clip_ratio/high_mean": 0.005394282125052996, "clip_ratio/low_mean": 0.0011522273453010712, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006546509452164173, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 864.0, "completions/mean_length": 735.5, "completions/mean_terminated_length": 580.1538696289062, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.12399293575435877, "epoch": 0.20648464163822525, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.028441090136766434, "kl": 0.011942001816350967, "learning_rate": 7.1467576791808875e-06, "loss": 0.0492, "num_tokens": 38859212.0, "reward": 1.25, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.2704269886016846, "sampling/importance_sampling_ratio/mean": 0.40470609068870544, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.313950538635254, "sampling/sampling_logp_difference/mean": 0.0149247320368886, "step": 363, "step_time": 75.89735764241777 }, { "clip_ratio/high_max": 0.012363912363070995, "clip_ratio/high_mean": 0.004360162405646406, "clip_ratio/low_mean": 0.0032980774994939566, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007658239977899939, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 832.7750244140625, "completions/mean_terminated_length": 641.5499877929688, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.11803748272359371, "epoch": 0.20705346985210465, "frac_reward_zero_std": 0.0, "grad_norm": 0.03010200709104538, "kl": 0.012103458400815725, "learning_rate": 7.141638225255973e-06, "loss": 0.0413, "num_tokens": 38958467.0, "reward": 0.9000000357627869, "reward_std": 0.9554163813591003, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.9519875049591064, "sampling/importance_sampling_ratio/mean": 0.44783011078834534, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9299843311309814, "sampling/sampling_logp_difference/mean": 0.013601789250969887, "step": 364, "step_time": 73.28522314224392 }, { "clip_ratio/high_max": 0.01641556911636144, "clip_ratio/high_mean": 0.004983001475920901, "clip_ratio/low_mean": 0.0014766091335332021, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006459610711317509, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 950.0, "completions/mean_length": 677.2999877929688, "completions/mean_terminated_length": 469.2799987792969, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.07731610769405961, "epoch": 0.20762229806598406, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019675346091389656, "kl": 0.012317910848651081, "learning_rate": 7.1365187713310575e-06, "loss": 0.014, "num_tokens": 39053911.0, "reward": 1.149999976158142, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.2053885459899902, "sampling/importance_sampling_ratio/mean": 0.47714701294898987, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.445345401763916, "sampling/sampling_logp_difference/mean": 0.011438167653977871, "step": 365, "step_time": 70.71718229562975 }, { "clip_ratio/high_max": 0.019644207903183997, "clip_ratio/high_mean": 0.007996941974852234, "clip_ratio/low_mean": 0.0010009765755967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008997918630484492, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 706.75, "completions/mean_terminated_length": 554.0, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.09882117435336113, "epoch": 0.20819112627986347, "frac_reward_zero_std": 0.0, "grad_norm": 0.04864831641316414, "kl": 0.013863707543350756, "learning_rate": 7.131399317406143e-06, "loss": -0.0804, "num_tokens": 39150525.0, "reward": 1.350000023841858, "reward_std": 0.9486833214759827, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.4968996047973633, "sampling/importance_sampling_ratio/mean": 0.6783168911933899, "sampling/importance_sampling_ratio/min": 0.0026512828189879656, "sampling/sampling_logp_difference/max": 2.388848304748535, "sampling/sampling_logp_difference/mean": 0.01229773461818695, "step": 366, "step_time": 71.61247110716067 }, { "clip_ratio/high_max": 0.010309357894584537, "clip_ratio/high_mean": 0.002794643645756878, "clip_ratio/low_mean": 0.0015869140552240424, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004381557693704963, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 810.1000366210938, "completions/mean_terminated_length": 681.760009765625, "completions/min_length": 371.0, "completions/min_terminated_length": 371.0, "entropy": 0.09506783308461308, "epoch": 0.20875995449374288, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.027159275487065315, "kl": 0.011357683397363871, "learning_rate": 7.126279863481229e-06, "loss": 0.1014, "num_tokens": 39259329.0, "reward": 1.25, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.8202977180480957, "sampling/importance_sampling_ratio/mean": 0.41826003789901733, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3554415702819824, "sampling/sampling_logp_difference/mean": 0.012799267657101154, "step": 367, "step_time": 73.41691387654282 }, { "clip_ratio/high_max": 0.015123661258257926, "clip_ratio/high_mean": 0.00601385225309059, "clip_ratio/low_mean": 0.001788736968592275, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078025892144069076, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 963.0, "completions/mean_length": 819.7750244140625, "completions/mean_terminated_length": 697.239990234375, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "entropy": 0.12900406308472157, "epoch": 0.20932878270762229, "frac_reward_zero_std": 0.0, "grad_norm": 0.03836379572749138, "kl": 0.014431760529987514, "learning_rate": 7.121160409556315e-06, "loss": -0.1783, "num_tokens": 39354240.0, "reward": 1.100000023841858, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.416776418685913, "sampling/importance_sampling_ratio/mean": 0.43811970949172974, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0141258239746094, "sampling/sampling_logp_difference/mean": 0.015331277623772621, "step": 368, "step_time": 70.25435513537377 }, { "clip_ratio/high_max": 0.008946708752773702, "clip_ratio/high_mean": 0.0021655313030350953, "clip_ratio/low_mean": 0.0016357421991415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038012733857613057, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 891.0, "completions/mean_length": 855.6749877929688, "completions/mean_terminated_length": 603.1875, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "entropy": 0.10817428771406412, "epoch": 0.2098976109215017, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016693370416760445, "kl": 0.014416408375836909, "learning_rate": 7.116040955631399e-06, "loss": 0.0195, "num_tokens": 39468603.0, "reward": 0.75, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.5886286497116089, "sampling/importance_sampling_ratio/mean": 0.29033157229423523, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.500070571899414, "sampling/sampling_logp_difference/mean": 0.013264867477118969, "step": 369, "step_time": 75.77292598434724 }, { "clip_ratio/high_max": 0.018167239148169756, "clip_ratio/high_mean": 0.006117551645729691, "clip_ratio/low_mean": 0.0012695312616415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007387082790955901, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 917.125, "completions/mean_terminated_length": 756.8125, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.16179404687136412, "epoch": 0.21046643913538113, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.026392649859189987, "kl": 0.014496060030069202, "learning_rate": 7.110921501706485e-06, "loss": 0.04, "num_tokens": 39572040.0, "reward": 0.625, "reward_std": 0.8378084897994995, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.1445369720458984, "sampling/importance_sampling_ratio/mean": 0.38574090600013733, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.704411506652832, "sampling/sampling_logp_difference/mean": 0.016825499013066292, "step": 370, "step_time": 71.8025153905619 }, { "clip_ratio/high_max": 0.012280952534638345, "clip_ratio/high_mean": 0.003652033119578846, "clip_ratio/low_mean": 0.0014648437809228199, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005116876971442252, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 934.7750244140625, "completions/mean_terminated_length": 749.4615478515625, "completions/min_length": 445.0, "completions/min_terminated_length": 445.0, "entropy": 0.16948716714978218, "epoch": 0.21103526734926054, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02617894671857357, "kl": 0.014262310462072492, "learning_rate": 7.10580204778157e-06, "loss": -0.0173, "num_tokens": 39682727.0, "reward": 0.6500000357627869, "reward_std": 0.9486833214759827, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 1.906677007675171, "sampling/importance_sampling_ratio/mean": 0.29583215713500977, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.80963397026062, "sampling/sampling_logp_difference/mean": 0.01707916520535946, "step": 371, "step_time": 75.26332169189118 }, { "clip_ratio/high_max": 0.005202054977416992, "clip_ratio/high_mean": 0.0010404109780211002, "clip_ratio/low_mean": 0.0012451172078726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022855281858937815, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 778.25, "completions/mean_terminated_length": 532.5, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.09445786010473967, "epoch": 0.21160409556313994, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.017440322786569595, "kl": 0.009627519000787288, "learning_rate": 7.1006825938566556e-06, "loss": -0.0305, "num_tokens": 39789353.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.555816650390625, "sampling/importance_sampling_ratio/mean": 0.5457619428634644, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.386834144592285, "sampling/sampling_logp_difference/mean": 0.013129833154380322, "step": 372, "step_time": 71.18544349749573 }, { "clip_ratio/high_max": 0.015967861749231815, "clip_ratio/high_mean": 0.0061242092633619905, "clip_ratio/low_mean": 0.0019706572347786278, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008094866352621466, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 741.9500122070312, "completions/mean_terminated_length": 511.18182373046875, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.10604496207088232, "epoch": 0.21217292377701935, "frac_reward_zero_std": 0.0, "grad_norm": 0.034667640924453735, "kl": 0.017806296818889678, "learning_rate": 7.09556313993174e-06, "loss": 0.2652, "num_tokens": 39894887.0, "reward": 1.024999976158142, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.2036936283111572, "sampling/importance_sampling_ratio/mean": 0.47466355562210083, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.4381961822509766, "sampling/sampling_logp_difference/mean": 0.013463245704770088, "step": 373, "step_time": 73.42671752395108 }, { "clip_ratio/high_max": 0.012778421863913536, "clip_ratio/high_mean": 0.003982354101026431, "clip_ratio/low_mean": 0.003344726559589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007327080704271793, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 869.2000122070312, "completions/mean_terminated_length": 508.0, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.1806161981076002, "epoch": 0.21274175199089876, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.010221281088888645, "kl": 0.016056295367889106, "learning_rate": 7.0904436860068264e-06, "loss": -0.011, "num_tokens": 40004671.0, "reward": 0.5250000357627869, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.9423410892486572, "sampling/importance_sampling_ratio/mean": 0.37650376558303833, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6921324729919434, "sampling/sampling_logp_difference/mean": 0.01767025701701641, "step": 374, "step_time": 74.78168795444071 }, { "clip_ratio/high_max": 0.010872511600609869, "clip_ratio/high_mean": 0.002662206970853731, "clip_ratio/low_mean": 0.0014648437354480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004127050677197985, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 877.7750244140625, "completions/mean_terminated_length": 439.1000061035156, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.14651880785822868, "epoch": 0.21331058020477817, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022704914212226868, "kl": 0.012541215226519853, "learning_rate": 7.085324232081912e-06, "loss": -0.0151, "num_tokens": 40113430.0, "reward": 0.45000001788139343, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.7992584705352783, "sampling/importance_sampling_ratio/mean": 0.450876384973526, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.30530047416687, "sampling/sampling_logp_difference/mean": 0.014642494730651379, "step": 375, "step_time": 72.49681227607653 }, { "clip_ratio/high_max": 0.00397300097392872, "clip_ratio/high_mean": 0.0010469293993082829, "clip_ratio/low_mean": 0.0018310546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028779840795323253, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 763.25, "completions/mean_terminated_length": 549.9091186523438, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.12111387588083744, "epoch": 0.21387940841865757, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02737320028245449, "kl": 0.01311986381188035, "learning_rate": 7.080204778156997e-06, "loss": 0.0551, "num_tokens": 40210016.0, "reward": 1.0500000715255737, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.365137815475464, "sampling/importance_sampling_ratio/mean": 0.4549511969089508, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8126122951507568, "sampling/sampling_logp_difference/mean": 0.01507710199803114, "step": 376, "step_time": 69.99973479937762 }, { "clip_ratio/high_max": 0.012073576042894274, "clip_ratio/high_mean": 0.0036160669988021255, "clip_ratio/low_mean": 0.002148437506548362, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005764504487160593, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 958.0, "completions/mean_length": 914.7000122070312, "completions/mean_terminated_length": 659.6666870117188, "completions/min_length": 485.0, "completions/min_terminated_length": 485.0, "entropy": 0.1763063594698906, "epoch": 0.21444823663253698, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019677970558404922, "kl": 0.012059074593707919, "learning_rate": 7.075085324232082e-06, "loss": 0.0439, "num_tokens": 40320460.0, "reward": 0.4000000059604645, "reward_std": 0.6717752814292908, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.9923250675201416, "sampling/importance_sampling_ratio/mean": 0.704576849937439, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3166980743408203, "sampling/sampling_logp_difference/mean": 0.017221085727214813, "step": 377, "step_time": 74.08186792605557 }, { "clip_ratio/high_max": 0.023040580679662526, "clip_ratio/high_mean": 0.0077371602528728545, "clip_ratio/low_mean": 0.0021621989108098205, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009899359138216823, "completions/clipped_ratio": 0.17499999701976776, "completions/max_length": 1024.0, "completions/max_terminated_length": 929.0, "completions/mean_length": 552.125, "completions/mean_terminated_length": 452.0303039550781, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.12778887432068586, "epoch": 0.2150170648464164, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.028075695037841797, "kl": 0.018618810339830816, "learning_rate": 7.069965870307167e-06, "loss": 0.1238, "num_tokens": 40412665.0, "reward": 1.4500000476837158, "reward_std": 0.7828285694122314, "rewards/format_reward/mean": 0.824999988079071, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.8560426235198975, "sampling/importance_sampling_ratio/mean": 0.3918724060058594, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.974318265914917, "sampling/sampling_logp_difference/mean": 0.016971595585346222, "step": 378, "step_time": 67.15972757083364 }, { "clip_ratio/high_max": 0.011350994638632983, "clip_ratio/high_mean": 0.0030129377701086923, "clip_ratio/low_mean": 0.002172851556679234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005185789399547502, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 923.2999877929688, "completions/mean_terminated_length": 714.1538696289062, "completions/min_length": 446.0, "completions/min_terminated_length": 446.0, "entropy": 0.11594932153820992, "epoch": 0.2155858930602958, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.2835873067378998, "kl": 0.03221559553639963, "learning_rate": 7.064846416382253e-06, "loss": 0.0824, "num_tokens": 40526533.0, "reward": 0.5250000357627869, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.1252658367156982, "sampling/importance_sampling_ratio/mean": 0.35512274503707886, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.758603572845459, "sampling/sampling_logp_difference/mean": 0.012943433597683907, "step": 379, "step_time": 75.51728792884387 }, { "clip_ratio/high_max": 0.01901487202849239, "clip_ratio/high_mean": 0.007245217188028619, "clip_ratio/low_mean": 0.0020751953197759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009320412762463093, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 882.0, "completions/mean_length": 755.0499877929688, "completions/mean_terminated_length": 486.1000061035156, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "entropy": 0.10897838231176138, "epoch": 0.2161547212741752, "frac_reward_zero_std": 0.0, "grad_norm": 0.061747241765260696, "kl": 0.034216373111121356, "learning_rate": 7.059726962457338e-06, "loss": 0.0399, "num_tokens": 40628847.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.118907928466797, "sampling/importance_sampling_ratio/mean": 0.34592097997665405, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.171541213989258, "sampling/sampling_logp_difference/mean": 0.014188903383910656, "step": 380, "step_time": 72.28314927080646 }, { "clip_ratio/high_max": 0.007484135567210615, "clip_ratio/high_mean": 0.003026559963473119, "clip_ratio/low_mean": 0.002233032777439803, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005259592697257176, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 906.0, "completions/mean_length": 845.2999877929688, "completions/mean_terminated_length": 547.4666748046875, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.11904501263052225, "epoch": 0.2167235494880546, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.032222189009189606, "kl": 0.017364352359436452, "learning_rate": 7.054607508532423e-06, "loss": -0.0017, "num_tokens": 40729987.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.669055938720703, "sampling/importance_sampling_ratio/mean": 0.3218509554862976, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3417041301727295, "sampling/sampling_logp_difference/mean": 0.014133351854979992, "step": 381, "step_time": 72.66064281528816 }, { "clip_ratio/high_max": 0.007061493466608226, "clip_ratio/high_mean": 0.002057498277281411, "clip_ratio/low_mean": 0.004077148390933871, "clip_ratio/low_min": 0.0009765625, "clip_ratio/region_mean": 0.006134646770078689, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 939.125, "completions/mean_terminated_length": 646.7777709960938, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "entropy": 0.1254437705501914, "epoch": 0.21729237770193402, "frac_reward_zero_std": 0.0, "grad_norm": 0.027250753715634346, "kl": 0.010780929296743125, "learning_rate": 7.049488054607509e-06, "loss": 0.0128, "num_tokens": 40842376.0, "reward": 0.45000001788139343, "reward_std": 0.8458041548728943, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.221113681793213, "sampling/importance_sampling_ratio/mean": 0.37180599570274353, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.099139451980591, "sampling/sampling_logp_difference/mean": 0.01319982297718525, "step": 382, "step_time": 75.4614151448477 }, { "clip_ratio/high_max": 0.009028137312270701, "clip_ratio/high_mean": 0.0024619625182822347, "clip_ratio/low_mean": 0.000561523440410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030234859441407025, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 699.0250244140625, "completions/mean_terminated_length": 559.75, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.14243170246481895, "epoch": 0.21786120591581343, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.020854752510786057, "kl": 0.01386875077150762, "learning_rate": 7.0443686006825945e-06, "loss": -0.0817, "num_tokens": 40942617.0, "reward": 1.399999976158142, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.699999988079071, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.461381435394287, "sampling/importance_sampling_ratio/mean": 0.4864066243171692, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2277188301086426, "sampling/sampling_logp_difference/mean": 0.015771348029375076, "step": 383, "step_time": 72.60333753819577 }, { "clip_ratio/high_max": 0.008284245268441737, "clip_ratio/high_mean": 0.002509220561478287, "clip_ratio/low_mean": 0.0021736674461863004, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004682888014940545, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 977.0, "completions/mean_length": 714.6749877929688, "completions/mean_terminated_length": 565.74072265625, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.1265179943293333, "epoch": 0.21843003412969283, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0158549714833498, "kl": 0.020805081934668124, "learning_rate": 7.03924914675768e-06, "loss": 0.0439, "num_tokens": 41049876.0, "reward": 1.0750000476837158, "reward_std": 0.8589648008346558, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.8222336769104004, "sampling/importance_sampling_ratio/mean": 0.44019633531570435, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9103856086730957, "sampling/sampling_logp_difference/mean": 0.015662260353565216, "step": 384, "step_time": 73.38307248218916 }, { "clip_ratio/high_max": 0.012562151649035513, "clip_ratio/high_mean": 0.004814385931240395, "clip_ratio/low_mean": 0.0021484374956344254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006962823390495032, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 811.125, "completions/mean_terminated_length": 636.95458984375, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "entropy": 0.12365886103361845, "epoch": 0.21899886234357224, "frac_reward_zero_std": 0.0, "grad_norm": 0.029274500906467438, "kl": 0.011529407231137156, "learning_rate": 7.0341296928327645e-06, "loss": -0.0158, "num_tokens": 41154033.0, "reward": 1.0500000715255737, "reward_std": 0.9857965111732483, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.311209201812744, "sampling/importance_sampling_ratio/mean": 0.3881289064884186, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3137543201446533, "sampling/sampling_logp_difference/mean": 0.014330494217574596, "step": 385, "step_time": 71.6427995574195 }, { "clip_ratio/high_max": 0.009914669906720519, "clip_ratio/high_mean": 0.002962430560728535, "clip_ratio/low_mean": 0.002197265625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005159696098417044, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 819.0, "completions/mean_length": 905.9000244140625, "completions/mean_terminated_length": 594.5454711914062, "completions/min_length": 394.0, "completions/min_terminated_length": 394.0, "entropy": 0.12247486971318722, "epoch": 0.21956769055745165, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02505488134920597, "kl": 0.008290158410090953, "learning_rate": 7.02901023890785e-06, "loss": 0.1011, "num_tokens": 41262221.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.5931284427642822, "sampling/importance_sampling_ratio/mean": 0.47628384828567505, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6462960243225098, "sampling/sampling_logp_difference/mean": 0.012659120373427868, "step": 386, "step_time": 73.45465328218415 }, { "clip_ratio/high_max": 0.011856027238536626, "clip_ratio/high_mean": 0.004179105468210764, "clip_ratio/low_mean": 0.003328734092065133, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.007507839414756745, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 836.1500244140625, "completions/mean_terminated_length": 648.2999877929688, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.10593572445213795, "epoch": 0.22013651877133106, "frac_reward_zero_std": 0.0, "grad_norm": 0.03667483106255531, "kl": 0.009465521667152643, "learning_rate": 7.023890784982935e-06, "loss": 0.0726, "num_tokens": 41373131.0, "reward": 0.875, "reward_std": 0.9388344883918762, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.92921781539917, "sampling/importance_sampling_ratio/mean": 0.6446430087089539, "sampling/importance_sampling_ratio/min": 0.0060999831184744835, "sampling/sampling_logp_difference/max": 1.5240285396575928, "sampling/sampling_logp_difference/mean": 0.01255376823246479, "step": 387, "step_time": 79.21875636419281 }, { "clip_ratio/high_max": 0.012032175553031266, "clip_ratio/high_mean": 0.002898854436352849, "clip_ratio/low_mean": 0.0020720821048598737, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004970936541212723, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 911.0, "completions/mean_length": 886.2250366210938, "completions/mean_terminated_length": 472.8999938964844, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.13272469211369753, "epoch": 0.22070534698521047, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02545480988919735, "kl": 0.008439406054094434, "learning_rate": 7.01877133105802e-06, "loss": -0.0495, "num_tokens": 41473588.0, "reward": 0.4749999940395355, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.7667951583862305, "sampling/importance_sampling_ratio/mean": 0.4307376742362976, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1511693000793457, "sampling/sampling_logp_difference/mean": 0.014758973382413387, "step": 388, "step_time": 72.16609926777892 }, { "clip_ratio/high_max": 0.011093478533439338, "clip_ratio/high_mean": 0.0034276694495929405, "clip_ratio/low_mean": 0.0018007587023021188, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005228428024565801, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 903.4750366210938, "completions/mean_terminated_length": 679.6428833007812, "completions/min_length": 441.0, "completions/min_terminated_length": 441.0, "entropy": 0.0842977361753583, "epoch": 0.22127417519908987, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015405195765197277, "kl": 0.007910866348538548, "learning_rate": 7.013651877133105e-06, "loss": 0.0704, "num_tokens": 41580503.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.3514816761016846, "sampling/importance_sampling_ratio/mean": 0.354369580745697, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7347203493118286, "sampling/sampling_logp_difference/mean": 0.009943424724042416, "step": 389, "step_time": 72.10362727777101 }, { "clip_ratio/high_max": 0.004980272613465786, "clip_ratio/high_mean": 0.0012546751968329772, "clip_ratio/low_mean": 0.0010742187587311491, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0023288939482881688, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 889.0, "completions/mean_length": 855.1749877929688, "completions/mean_terminated_length": 461.25, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.08254070021212101, "epoch": 0.22184300341296928, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.02220100909471512, "kl": 0.006803377240430564, "learning_rate": 7.008532423208192e-06, "loss": 0.1019, "num_tokens": 41679670.0, "reward": 0.6000000238418579, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.541884422302246, "sampling/importance_sampling_ratio/mean": 0.46431875228881836, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.892815351486206, "sampling/sampling_logp_difference/mean": 0.010893726721405983, "step": 390, "step_time": 70.15185317955911 }, { "clip_ratio/high_max": 0.012709191360045224, "clip_ratio/high_mean": 0.004435957205714658, "clip_ratio/low_mean": 0.001196289074869128, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005632246291497722, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 978.0, "completions/mean_length": 920.5250244140625, "completions/mean_terminated_length": 728.357177734375, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "entropy": 0.1095990058965981, "epoch": 0.2224118316268487, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015090777538716793, "kl": 0.008435190888121724, "learning_rate": 7.003412969283277e-06, "loss": 0.0336, "num_tokens": 41784523.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.1973776817321777, "sampling/importance_sampling_ratio/mean": 0.3597260117530823, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0783424377441406, "sampling/sampling_logp_difference/mean": 0.012769750319421291, "step": 391, "step_time": 72.28936938894913 }, { "clip_ratio/high_max": 0.010039901128038764, "clip_ratio/high_mean": 0.002579629101091996, "clip_ratio/low_mean": 0.0020996093589928932, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0046792383946012706, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 917.875, "completions/mean_terminated_length": 670.25, "completions/min_length": 441.0, "completions/min_terminated_length": 441.0, "entropy": 0.10435466095805168, "epoch": 0.2229806598407281, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03960483521223068, "kl": 0.008881088288035244, "learning_rate": 6.998293515358362e-06, "loss": -0.116, "num_tokens": 41892046.0, "reward": 0.375, "reward_std": 0.6278780102729797, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.3092920780181885, "sampling/importance_sampling_ratio/mean": 0.4541400372982025, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.80963397026062, "sampling/sampling_logp_difference/mean": 0.011579856276512146, "step": 392, "step_time": 72.30278184823692 }, { "clip_ratio/high_max": 0.01541635231114924, "clip_ratio/high_mean": 0.006263440154725686, "clip_ratio/low_mean": 0.0017578124752617441, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00802125281188637, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 819.875, "completions/mean_terminated_length": 635.1904907226562, "completions/min_length": 420.0, "completions/min_terminated_length": 420.0, "entropy": 0.09827239718288183, "epoch": 0.2235494880546075, "frac_reward_zero_std": 0.0, "grad_norm": 0.06895232945680618, "kl": 0.012546011246740818, "learning_rate": 6.993174061433447e-06, "loss": -0.1048, "num_tokens": 42001881.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.831122398376465, "sampling/importance_sampling_ratio/mean": 0.6413580775260925, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0480151176452637, "sampling/sampling_logp_difference/mean": 0.012910827994346619, "step": 393, "step_time": 73.88408607570454 }, { "clip_ratio/high_max": 0.011174227460287511, "clip_ratio/high_mean": 0.0035576493537519127, "clip_ratio/low_mean": 0.0013570171868195757, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004914666526019573, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 788.2000122070312, "completions/mean_terminated_length": 574.857177734375, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.12489651143550873, "epoch": 0.2241183162684869, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02945665642619133, "kl": 0.011382492491975427, "learning_rate": 6.9880546075085326e-06, "loss": -0.0552, "num_tokens": 42105937.0, "reward": 1.024999976158142, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.7977585792541504, "sampling/importance_sampling_ratio/mean": 0.6773412823677063, "sampling/importance_sampling_ratio/min": 0.010086147114634514, "sampling/sampling_logp_difference/max": 2.019197463989258, "sampling/sampling_logp_difference/mean": 0.013812052085995674, "step": 394, "step_time": 69.85867267195135 }, { "clip_ratio/high_max": 0.010946404887363315, "clip_ratio/high_mean": 0.004258836808730848, "clip_ratio/low_mean": 0.0026855468895519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006944383727386594, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 929.6000366210938, "completions/mean_terminated_length": 835.2000122070312, "completions/min_length": 562.0, "completions/min_terminated_length": 562.0, "entropy": 0.13651888072490692, "epoch": 0.22468714448236632, "frac_reward_zero_std": 0.0, "grad_norm": 0.013508064672350883, "kl": 0.008601028239354491, "learning_rate": 6.982935153583618e-06, "loss": 0.0718, "num_tokens": 42224009.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.4010804891586304, "sampling/importance_sampling_ratio/mean": 0.22507281601428986, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8843969106674194, "sampling/sampling_logp_difference/mean": 0.01404484547674656, "step": 395, "step_time": 80.59840795747004 }, { "clip_ratio/high_max": 0.013087535393424332, "clip_ratio/high_mean": 0.004249871039064601, "clip_ratio/low_mean": 0.0009765625018189894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005226433597272262, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 759.4000244140625, "completions/mean_terminated_length": 520.0, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.08008899027481675, "epoch": 0.22525597269624573, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.020364195108413696, "kl": 0.010113902040757239, "learning_rate": 6.977815699658703e-06, "loss": -0.0478, "num_tokens": 42323641.0, "reward": 0.8500000238418579, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 1.8815480470657349, "sampling/importance_sampling_ratio/mean": 0.4514757990837097, "sampling/importance_sampling_ratio/min": 0.013277152553200722, "sampling/sampling_logp_difference/max": 2.552175521850586, "sampling/sampling_logp_difference/mean": 0.011029217392206192, "step": 396, "step_time": 71.40289733815007 }, { "clip_ratio/high_max": 0.010051974677480757, "clip_ratio/high_mean": 0.00228572532068938, "clip_ratio/low_mean": 0.001855468755820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004141194047406316, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 882.2000122070312, "completions/mean_terminated_length": 393.77777099609375, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.13272885233163834, "epoch": 0.22582480091012513, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024781687185168266, "kl": 0.015598621510434896, "learning_rate": 6.972696245733788e-06, "loss": 0.0003, "num_tokens": 42431017.0, "reward": 0.45000001788139343, "reward_std": 0.8458041548728943, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.9308977127075195, "sampling/importance_sampling_ratio/mean": 0.5477339029312134, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9825890064239502, "sampling/sampling_logp_difference/mean": 0.013844961300492287, "step": 397, "step_time": 73.40949233062565 }, { "clip_ratio/high_max": 0.010375719575677067, "clip_ratio/high_mean": 0.0030169520468916744, "clip_ratio/low_mean": 0.0022705078445142135, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005287459935061634, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 990.0, "completions/mean_length": 906.0, "completions/mean_terminated_length": 594.9091186523438, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "entropy": 0.07844050787389278, "epoch": 0.22639362912400454, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018607674166560173, "kl": 0.006159231124911457, "learning_rate": 6.967576791808874e-06, "loss": 0.1125, "num_tokens": 42531985.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.579343318939209, "sampling/importance_sampling_ratio/mean": 0.6182876825332642, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1848351955413818, "sampling/sampling_logp_difference/mean": 0.008515560999512672, "step": 398, "step_time": 72.64386452967301 }, { "clip_ratio/high_max": 0.014445448061451316, "clip_ratio/high_mean": 0.0036700372584164143, "clip_ratio/low_mean": 0.0017578124861756805, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005427849740954116, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 903.7000122070312, "completions/mean_terminated_length": 542.7999877929688, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.11193892545998096, "epoch": 0.22696245733788395, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0398605614900589, "kl": 0.009737046202644706, "learning_rate": 6.96245733788396e-06, "loss": -0.0449, "num_tokens": 42634869.0, "reward": 0.3499999940395355, "reward_std": 0.6621642708778381, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.791365623474121, "sampling/importance_sampling_ratio/mean": 0.45175981521606445, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.903963565826416, "sampling/sampling_logp_difference/mean": 0.01167607493698597, "step": 399, "step_time": 73.08434686716646 }, { "clip_ratio/high_max": 0.01563854521373287, "clip_ratio/high_mean": 0.003972587961470708, "clip_ratio/low_mean": 0.0006709346052957699, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0046435225231107324, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 818.6749877929688, "completions/mean_terminated_length": 591.73681640625, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.135028806515038, "epoch": 0.22753128555176336, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01682640239596367, "kl": 0.011974377615842968, "learning_rate": 6.957337883959044e-06, "loss": -0.0364, "num_tokens": 42735568.0, "reward": 0.925000011920929, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.511049747467041, "sampling/importance_sampling_ratio/mean": 0.4982127547264099, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5458014011383057, "sampling/sampling_logp_difference/mean": 0.014806347899138927, "step": 400, "step_time": 72.64607441541739 }, { "clip_ratio/high_max": 0.015842822729609907, "clip_ratio/high_mean": 0.003926875127945095, "clip_ratio/low_mean": 0.0033203124767169356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007247187662869692, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 900.8250122070312, "completions/mean_terminated_length": 531.2999877929688, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "entropy": 0.11252890806645155, "epoch": 0.22810011376564276, "frac_reward_zero_std": 0.0, "grad_norm": 0.032855551689863205, "kl": 0.009127261175308377, "learning_rate": 6.95221843003413e-06, "loss": 0.0168, "num_tokens": 42848721.0, "reward": 0.5, "reward_std": 0.8770580291748047, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.8757755756378174, "sampling/importance_sampling_ratio/mean": 0.4467374384403229, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9130585193634033, "sampling/sampling_logp_difference/mean": 0.0123746981844306, "step": 401, "step_time": 76.24662977340631 }, { "clip_ratio/high_max": 0.009617836796678603, "clip_ratio/high_mean": 0.002950561261968687, "clip_ratio/low_mean": 0.0033935546525754035, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006344115827232599, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 966.5, "completions/mean_terminated_length": 768.4444580078125, "completions/min_length": 543.0, "completions/min_terminated_length": 543.0, "entropy": 0.14611777383834124, "epoch": 0.22866894197952217, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.017649715766310692, "kl": 0.009311897330917418, "learning_rate": 6.947098976109215e-06, "loss": -0.013, "num_tokens": 42958549.0, "reward": 0.45000001788139343, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 1.998187780380249, "sampling/importance_sampling_ratio/mean": 0.3924705386161804, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8376283645629883, "sampling/sampling_logp_difference/mean": 0.014236216433346272, "step": 402, "step_time": 73.99083024077117 }, { "clip_ratio/high_max": 0.017855827347375453, "clip_ratio/high_mean": 0.005802681800560094, "clip_ratio/low_mean": 0.0009033203241415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006706002110149711, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 874.1000366210938, "completions/mean_terminated_length": 671.2941284179688, "completions/min_length": 460.0, "completions/min_terminated_length": 460.0, "entropy": 0.17095314990729094, "epoch": 0.22923777019340158, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0393192321062088, "kl": 0.01106576039455831, "learning_rate": 6.941979522184301e-06, "loss": -0.0026, "num_tokens": 43061993.0, "reward": 0.7250000238418579, "reward_std": 0.905467689037323, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.9111647605895996, "sampling/importance_sampling_ratio/mean": 0.4285149574279785, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5857360363006592, "sampling/sampling_logp_difference/mean": 0.017242079600691795, "step": 403, "step_time": 71.9414350814186 }, { "clip_ratio/high_max": 0.006313195131951943, "clip_ratio/high_mean": 0.0017216408523381688, "clip_ratio/low_mean": 0.001904296877910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036259376065572724, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 942.6749877929688, "completions/mean_terminated_length": 662.5555419921875, "completions/min_length": 507.0, "completions/min_terminated_length": 507.0, "entropy": 0.08702331967651844, "epoch": 0.229806598407281, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0326058566570282, "kl": 0.008518923597875983, "learning_rate": 6.936860068259385e-06, "loss": 0.1437, "num_tokens": 43175580.0, "reward": 0.45000001788139343, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290207743644714, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.7557754516601562, "sampling/importance_sampling_ratio/mean": 0.7134428024291992, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.771540641784668, "sampling/sampling_logp_difference/mean": 0.010287116281688213, "step": 404, "step_time": 76.49214334622957 }, { "clip_ratio/high_max": 0.007934875058708712, "clip_ratio/high_mean": 0.0019892964483005926, "clip_ratio/low_mean": 0.002148437502910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004137733834795654, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 961.7000122070312, "completions/mean_terminated_length": 668.0, "completions/min_length": 444.0, "completions/min_terminated_length": 444.0, "entropy": 0.07250558119267225, "epoch": 0.23037542662116042, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013921248726546764, "kl": 0.007304667378775775, "learning_rate": 6.931740614334471e-06, "loss": 0.0197, "num_tokens": 43284984.0, "reward": 0.3499999940395355, "reward_std": 0.7696152329444885, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.382617235183716, "sampling/importance_sampling_ratio/mean": 0.5132678747177124, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0369465351104736, "sampling/sampling_logp_difference/mean": 0.009084681048989296, "step": 405, "step_time": 72.92469088337384 }, { "clip_ratio/high_max": 0.014416339923627675, "clip_ratio/high_mean": 0.004022786364657804, "clip_ratio/low_mean": 0.001757812497089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005780598847195506, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 759.75, "completions/mean_terminated_length": 659.5172119140625, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.16729692369699478, "epoch": 0.23094425483503983, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02749565802514553, "kl": 0.012203378370031714, "learning_rate": 6.926621160409557e-06, "loss": 0.0224, "num_tokens": 43390070.0, "reward": 1.4500000476837158, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.7250000238418579, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.9011356830596924, "sampling/importance_sampling_ratio/mean": 0.4459919035434723, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.740882396697998, "sampling/sampling_logp_difference/mean": 0.01881447806954384, "step": 406, "step_time": 71.46422257646918 }, { "clip_ratio/high_max": 0.011623327329289168, "clip_ratio/high_mean": 0.004499506510910578, "clip_ratio/low_mean": 0.0007324218640860636, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005231928400462493, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 864.0, "completions/mean_length": 645.7000122070312, "completions/mean_terminated_length": 535.8709716796875, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.10969168692827225, "epoch": 0.23151308304891924, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01809731125831604, "kl": 0.012879647896625102, "learning_rate": 6.921501706484642e-06, "loss": -0.0005, "num_tokens": 43488218.0, "reward": 1.350000023841858, "reward_std": 0.8335896730422974, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.9200812578201294, "sampling/importance_sampling_ratio/mean": 0.353264719247818, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2563066482543945, "sampling/sampling_logp_difference/mean": 0.014249632135033607, "step": 407, "step_time": 69.36763216159306 }, { "clip_ratio/high_max": 0.010663622961146757, "clip_ratio/high_mean": 0.004180750918749254, "clip_ratio/low_mean": 0.0023581732239108533, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006538924062624574, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 978.0, "completions/mean_length": 793.9000244140625, "completions/mean_terminated_length": 585.7142944335938, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.11576312687247992, "epoch": 0.23208191126279865, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01929742470383644, "kl": 0.013316620257683098, "learning_rate": 6.916382252559727e-06, "loss": -0.0344, "num_tokens": 43594702.0, "reward": 0.9000000357627869, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.3271548748016357, "sampling/importance_sampling_ratio/mean": 0.4529818892478943, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.553633213043213, "sampling/sampling_logp_difference/mean": 0.01477885153144598, "step": 408, "step_time": 72.87474311050028 }, { "clip_ratio/high_max": 0.013211024575866759, "clip_ratio/high_mean": 0.004217666195472702, "clip_ratio/low_mean": 0.0017157017646241002, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00593336793826893, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 843.8500366210938, "completions/mean_terminated_length": 600.11767578125, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "entropy": 0.08177464315667748, "epoch": 0.23265073947667805, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019855240359902382, "kl": 0.013767957221716642, "learning_rate": 6.911262798634812e-06, "loss": -0.02, "num_tokens": 43705384.0, "reward": 0.7250000238418579, "reward_std": 0.905467689037323, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.3332090377807617, "sampling/importance_sampling_ratio/mean": 0.3675207197666168, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.309124708175659, "sampling/sampling_logp_difference/mean": 0.011260807514190674, "step": 409, "step_time": 75.1789313540794 }, { "clip_ratio/high_max": 0.009023472899571061, "clip_ratio/high_mean": 0.0019800101581495255, "clip_ratio/low_mean": 0.0020289306339691393, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004008940770290792, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 932.9249877929688, "completions/mean_terminated_length": 659.7000122070312, "completions/min_length": 453.0, "completions/min_terminated_length": 453.0, "entropy": 0.07440194906666875, "epoch": 0.23321956769055746, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02324696257710457, "kl": 0.008635096601210535, "learning_rate": 6.906143344709898e-06, "loss": -0.0051, "num_tokens": 43811005.0, "reward": 0.42500001192092896, "reward_std": 0.780778706073761, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.5563206672668457, "sampling/importance_sampling_ratio/mean": 0.33783406019210815, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.067509651184082, "sampling/sampling_logp_difference/mean": 0.009672543965280056, "step": 410, "step_time": 73.58065867773257 }, { "clip_ratio/high_max": 0.014985587098635733, "clip_ratio/high_mean": 0.006224912503967062, "clip_ratio/low_mean": 0.0024596344883320853, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008684546744916588, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 843.0, "completions/mean_length": 751.75, "completions/mean_terminated_length": 505.4285888671875, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.1270151687785983, "epoch": 0.23378839590443687, "frac_reward_zero_std": 0.0, "grad_norm": 0.038537003099918365, "kl": 0.01596217544283718, "learning_rate": 6.901023890784983e-06, "loss": -0.0924, "num_tokens": 43917523.0, "reward": 0.9000000357627869, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.679224967956543, "sampling/importance_sampling_ratio/mean": 0.5488106608390808, "sampling/importance_sampling_ratio/min": 0.0020034359768033028, "sampling/sampling_logp_difference/max": 2.141247272491455, "sampling/sampling_logp_difference/mean": 0.015037707053124905, "step": 411, "step_time": 70.6979507880751 }, { "clip_ratio/high_max": 0.02171174355316907, "clip_ratio/high_mean": 0.0071831970126368105, "clip_ratio/low_mean": 0.0017822265581344254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008965423738118261, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 818.4249877929688, "completions/mean_terminated_length": 510.0625, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.10615964233875275, "epoch": 0.23435722411831628, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.031737685203552246, "kl": 0.011118817375972867, "learning_rate": 6.895904436860068e-06, "loss": 0.0342, "num_tokens": 44028724.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.529306650161743, "sampling/importance_sampling_ratio/mean": 0.534620463848114, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0026445388793945, "sampling/sampling_logp_difference/mean": 0.013016799464821815, "step": 412, "step_time": 74.23679733881727 }, { "clip_ratio/high_max": 0.01137950731208548, "clip_ratio/high_mean": 0.003551629386493005, "clip_ratio/low_mean": 0.0012695312361756805, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004821160720894113, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 786.9000244140625, "completions/mean_terminated_length": 572.3809814453125, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.08702557068318129, "epoch": 0.23492605233219568, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03783963620662689, "kl": 0.009657595423050225, "learning_rate": 6.890784982935153e-06, "loss": 0.0315, "num_tokens": 44127824.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.646833658218384, "sampling/importance_sampling_ratio/mean": 0.5669244527816772, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.556399345397949, "sampling/sampling_logp_difference/mean": 0.011850639246404171, "step": 413, "step_time": 73.59200011449866 }, { "clip_ratio/high_max": 0.0071686970186419785, "clip_ratio/high_mean": 0.0017870459123514593, "clip_ratio/low_mean": 0.0016601562565483619, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003447202092502266, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 975.8250122070312, "completions/mean_terminated_length": 783.125, "completions/min_length": 540.0, "completions/min_terminated_length": 540.0, "entropy": 0.06804757891222835, "epoch": 0.2354948805460751, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03209605813026428, "kl": 0.00781931314850226, "learning_rate": 6.8856655290102396e-06, "loss": -0.0198, "num_tokens": 44225985.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.8925118446350098, "sampling/importance_sampling_ratio/mean": 0.5816173553466797, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 11.000015258789062, "sampling/sampling_logp_difference/mean": 0.009671995416283607, "step": 414, "step_time": 72.64847105741501 }, { "clip_ratio/high_max": 0.017055721255019307, "clip_ratio/high_mean": 0.004045212408527732, "clip_ratio/low_mean": 0.001969492484931834, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006014704908011481, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 925.0, "completions/mean_length": 753.9500122070312, "completions/mean_terminated_length": 455.47369384765625, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.12488736305385828, "epoch": 0.2360637087599545, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.034535255283117294, "kl": 0.013886310392990708, "learning_rate": 6.880546075085325e-06, "loss": -0.0584, "num_tokens": 44327703.0, "reward": 0.925000011920929, "reward_std": 0.9971113204956055, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.4010698795318604, "sampling/importance_sampling_ratio/mean": 0.4099365174770355, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0687332153320312, "sampling/sampling_logp_difference/mean": 0.015253255143761635, "step": 415, "step_time": 69.26823186175898 }, { "clip_ratio/high_max": 0.013331602327525616, "clip_ratio/high_mean": 0.002957018179586157, "clip_ratio/low_mean": 0.0020996093953726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005056627531303093, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 795.0, "completions/mean_length": 824.2000122070312, "completions/mean_terminated_length": 553.8823852539062, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.13257426116615534, "epoch": 0.2366325369738339, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04019419476389885, "kl": 0.009824086562730372, "learning_rate": 6.87542662116041e-06, "loss": -0.0947, "num_tokens": 44445023.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.4766149520874023, "sampling/importance_sampling_ratio/mean": 0.4765327572822571, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6049706935882568, "sampling/sampling_logp_difference/mean": 0.014687836170196533, "step": 416, "step_time": 75.67919608671218 }, { "clip_ratio/high_max": 0.011752174468711019, "clip_ratio/high_mean": 0.0032644736711517908, "clip_ratio/low_mean": 0.0025571357255103067, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005821609316626564, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 899.4750366210938, "completions/mean_terminated_length": 668.2142944335938, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "entropy": 0.10115108545869589, "epoch": 0.23720136518771331, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02056868001818657, "kl": 0.009763431851752102, "learning_rate": 6.870307167235495e-06, "loss": 0.0705, "num_tokens": 44539970.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.3172898292541504, "sampling/importance_sampling_ratio/mean": 0.5156133770942688, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4404759407043457, "sampling/sampling_logp_difference/mean": 0.01160463597625494, "step": 417, "step_time": 72.38808739488013 }, { "clip_ratio/high_max": 0.012448600202333182, "clip_ratio/high_mean": 0.003633257787441835, "clip_ratio/low_mean": 0.0014341001078719273, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005067357909865677, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 952.0, "completions/mean_length": 914.9750366210938, "completions/mean_terminated_length": 660.5833740234375, "completions/min_length": 423.0, "completions/min_terminated_length": 423.0, "entropy": 0.16215354576706886, "epoch": 0.23777019340159272, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01760532520711422, "kl": 0.009262458945158869, "learning_rate": 6.8651877133105804e-06, "loss": 0.0504, "num_tokens": 44655137.0, "reward": 0.5250000357627869, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.012118339538574, "sampling/importance_sampling_ratio/mean": 0.3402657210826874, "sampling/importance_sampling_ratio/min": 0.0029289263766258955, "sampling/sampling_logp_difference/max": 1.7483603954315186, "sampling/sampling_logp_difference/mean": 0.016174258664250374, "step": 418, "step_time": 76.46754199452698 }, { "clip_ratio/high_max": 0.01791922003030777, "clip_ratio/high_mean": 0.0072314330318477005, "clip_ratio/low_mean": 0.0010986328379658516, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008330065873451531, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 750.9249877929688, "completions/mean_terminated_length": 549.0869750976562, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.08551771007478237, "epoch": 0.23833902161547213, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.043367087841033936, "kl": 0.017376459552906454, "learning_rate": 6.860068259385666e-06, "loss": 0.004, "num_tokens": 44761118.0, "reward": 1.0, "reward_std": 0.9336995482444763, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.9362194538116455, "sampling/importance_sampling_ratio/mean": 0.4830195903778076, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.938668251037598, "sampling/sampling_logp_difference/mean": 0.01237501110881567, "step": 419, "step_time": 73.09379057609476 }, { "clip_ratio/high_max": 0.01186082960339263, "clip_ratio/high_mean": 0.0044080839870730415, "clip_ratio/low_mean": 0.002506501092284452, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006914585130289197, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 813.6500244140625, "completions/mean_terminated_length": 641.5454711914062, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.10429254453629255, "epoch": 0.23890784982935154, "frac_reward_zero_std": 0.0, "grad_norm": 0.03179411590099335, "kl": 0.01253380416892469, "learning_rate": 6.8549488054607505e-06, "loss": -0.0497, "num_tokens": 44855208.0, "reward": 1.0500000715255737, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.3578197956085205, "sampling/importance_sampling_ratio/mean": 0.40527573227882385, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.102463483810425, "sampling/sampling_logp_difference/mean": 0.013132250867784023, "step": 420, "step_time": 73.29807706130669 }, { "clip_ratio/high_max": 0.011892659589648247, "clip_ratio/high_mean": 0.005471458105603233, "clip_ratio/low_mean": 0.0018310546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00730251258937642, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 818.9000244140625, "completions/mean_terminated_length": 695.8399658203125, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.14278246834874153, "epoch": 0.23947667804323094, "frac_reward_zero_std": 0.0, "grad_norm": 0.025838488712906837, "kl": 0.014011244289577007, "learning_rate": 6.849829351535837e-06, "loss": 0.068, "num_tokens": 44955164.0, "reward": 1.2000000476837158, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.4518650770187378, "sampling/importance_sampling_ratio/mean": 0.2947077453136444, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.576721429824829, "sampling/sampling_logp_difference/mean": 0.014535866677761078, "step": 421, "step_time": 75.32114796992391 }, { "clip_ratio/high_max": 0.016467047040350735, "clip_ratio/high_mean": 0.006242547882720828, "clip_ratio/low_mean": 0.0015338766424974892, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007776424434268847, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 810.0, "completions/mean_length": 801.4249877929688, "completions/mean_terminated_length": 555.4210815429688, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.12599500641226768, "epoch": 0.24004550625711035, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.07061812281608582, "kl": 0.014991464966442436, "learning_rate": 6.844709897610922e-06, "loss": 0.11, "num_tokens": 45049541.0, "reward": 0.925000011920929, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.6089088916778564, "sampling/importance_sampling_ratio/mean": 0.5556357502937317, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.837857484817505, "sampling/sampling_logp_difference/mean": 0.014323624782264233, "step": 422, "step_time": 74.77607149095275 }, { "clip_ratio/high_max": 0.006260134978219867, "clip_ratio/high_mean": 0.0019844147318508476, "clip_ratio/low_mean": 0.002670048728759866, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004654463336919434, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 917.0, "completions/mean_length": 892.0250244140625, "completions/mean_terminated_length": 544.0909423828125, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.08875072002410889, "epoch": 0.24061433447098976, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01279648020863533, "kl": 0.00956316408701241, "learning_rate": 6.839590443686007e-06, "loss": 0.0912, "num_tokens": 45147638.0, "reward": 0.4000000059604645, "reward_std": 0.7089175581932068, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.500283718109131, "sampling/importance_sampling_ratio/mean": 0.3381464183330536, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.498194932937622, "sampling/sampling_logp_difference/mean": 0.011169278994202614, "step": 423, "step_time": 69.35221540625207 }, { "clip_ratio/high_max": 0.017847879382316023, "clip_ratio/high_mean": 0.0060395306936698034, "clip_ratio/low_mean": 0.0009172365971608087, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00695676717441529, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 920.0, "completions/mean_length": 735.1500244140625, "completions/mean_terminated_length": 611.357177734375, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.12582223676145077, "epoch": 0.24118316268486917, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.027193501591682434, "kl": 0.013261212967336178, "learning_rate": 6.834470989761092e-06, "loss": 0.0552, "num_tokens": 45251716.0, "reward": 1.3250000476837158, "reward_std": 0.9167249798774719, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.3750247955322266, "sampling/importance_sampling_ratio/mean": 0.4346339702606201, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0730791091918945, "sampling/sampling_logp_difference/mean": 0.014559024944901466, "step": 424, "step_time": 73.4493780194316 }, { "clip_ratio/high_max": 0.013121374533511698, "clip_ratio/high_mean": 0.0034447696525603533, "clip_ratio/low_mean": 0.002490234372089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005935003893682733, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 819.5250244140625, "completions/mean_terminated_length": 615.0499877929688, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.1091040838509798, "epoch": 0.24175199089874858, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0223085880279541, "kl": 0.012314420659095049, "learning_rate": 6.829351535836178e-06, "loss": 0.0113, "num_tokens": 45357209.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.974368095397949, "sampling/importance_sampling_ratio/mean": 0.6099594235420227, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8132424354553223, "sampling/sampling_logp_difference/mean": 0.013289756141602993, "step": 425, "step_time": 72.87341729085892 }, { "clip_ratio/high_max": 0.009924374287948012, "clip_ratio/high_mean": 0.0033187667868332937, "clip_ratio/low_mean": 0.0018310547020519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005149821372469887, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 834.0, "completions/mean_length": 738.4500122070312, "completions/mean_terminated_length": 548.0833740234375, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.1255155075341463, "epoch": 0.24232081911262798, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.05264278128743172, "kl": 0.01236278930446133, "learning_rate": 6.824232081911263e-06, "loss": 0.071, "num_tokens": 45463875.0, "reward": 1.024999976158142, "reward_std": 0.9195177555084229, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.0765206813812256, "sampling/importance_sampling_ratio/mean": 0.5326039791107178, "sampling/importance_sampling_ratio/min": 0.0050875344313681126, "sampling/sampling_logp_difference/max": 2.1365280151367188, "sampling/sampling_logp_difference/mean": 0.014755484648048878, "step": 426, "step_time": 72.20537421037443 }, { "clip_ratio/high_max": 0.011800344684161246, "clip_ratio/high_mean": 0.003785514272749424, "clip_ratio/low_mean": 0.003220932456315495, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007006446714513004, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 970.0, "completions/mean_length": 868.7750244140625, "completions/mean_terminated_length": 679.0555419921875, "completions/min_length": 443.0, "completions/min_terminated_length": 443.0, "entropy": 0.13524638954550028, "epoch": 0.2428896473265074, "frac_reward_zero_std": 0.0, "grad_norm": 0.023599309846758842, "kl": 0.011554497526958585, "learning_rate": 6.819112627986348e-06, "loss": -0.008, "num_tokens": 45556442.0, "reward": 0.8500000238418579, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.0539716482162476, "sampling/importance_sampling_ratio/mean": 0.2537735402584076, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.593803644180298, "sampling/sampling_logp_difference/mean": 0.014223475940525532, "step": 427, "step_time": 72.02873625513166 }, { "clip_ratio/high_max": 0.00901249417802319, "clip_ratio/high_mean": 0.002207784229540266, "clip_ratio/low_mean": 0.002221679675130872, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004429464024724439, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 840.6000366210938, "completions/mean_terminated_length": 637.8947143554688, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.1197702158242464, "epoch": 0.2434584755403868, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.029189441353082657, "kl": 0.011012446018867195, "learning_rate": 6.813993174061433e-06, "loss": 0.0693, "num_tokens": 45661642.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.9468433856964111, "sampling/importance_sampling_ratio/mean": 0.3248748779296875, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.236034631729126, "sampling/sampling_logp_difference/mean": 0.013622268103063107, "step": 428, "step_time": 74.7116903597489 }, { "clip_ratio/high_max": 0.009752926067449152, "clip_ratio/high_mean": 0.00280517711507855, "clip_ratio/low_mean": 0.001806640619179234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004611817799741402, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 797.0, "completions/mean_length": 922.75, "completions/mean_terminated_length": 574.0, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.1323143132030964, "epoch": 0.2440273037542662, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.043745867908000946, "kl": 0.010402434796560556, "learning_rate": 6.808873720136519e-06, "loss": 0.0503, "num_tokens": 45769312.0, "reward": 0.42500001192092896, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.9947402477264404, "sampling/importance_sampling_ratio/mean": 0.5912196040153503, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9538114070892334, "sampling/sampling_logp_difference/mean": 0.013784931972622871, "step": 429, "step_time": 74.35256916680373 }, { "clip_ratio/high_max": 0.012594676984008402, "clip_ratio/high_mean": 0.005059866234660149, "clip_ratio/low_mean": 0.002116750976711046, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.007176617335062474, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 849.0, "completions/mean_length": 789.875, "completions/mean_terminated_length": 555.75, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.08702374249696732, "epoch": 0.2445961319681456, "frac_reward_zero_std": 0.0, "grad_norm": 0.039792973548173904, "kl": 0.00932807318167761, "learning_rate": 6.803754266211605e-06, "loss": -0.0783, "num_tokens": 45876795.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.918605327606201, "sampling/importance_sampling_ratio/mean": 0.6139050722122192, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.222611904144287, "sampling/sampling_logp_difference/mean": 0.011648296378552914, "step": 430, "step_time": 74.50799898477271 }, { "clip_ratio/high_max": 0.010467647342011333, "clip_ratio/high_mean": 0.0037196026005403837, "clip_ratio/low_mean": 0.002233674007584341, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005953276762738824, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 903.6749877929688, "completions/mean_terminated_length": 740.8823852539062, "completions/min_length": 437.0, "completions/min_terminated_length": 437.0, "entropy": 0.15961748734116554, "epoch": 0.24516496018202502, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020180989056825638, "kl": 0.012090519303455949, "learning_rate": 6.798634812286689e-06, "loss": -0.0333, "num_tokens": 45983446.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.548309087753296, "sampling/importance_sampling_ratio/mean": 0.40516385436058044, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.215158224105835, "sampling/sampling_logp_difference/mean": 0.015605228021740913, "step": 431, "step_time": 74.69062241679057 }, { "clip_ratio/high_max": 0.009209097304847091, "clip_ratio/high_mean": 0.002257990519865416, "clip_ratio/low_mean": 0.0027832031046273187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005041193508077413, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 955.8250122070312, "completions/mean_terminated_length": 721.0, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.1330253230407834, "epoch": 0.24573378839590443, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020895542576909065, "kl": 0.00896791450213641, "learning_rate": 6.793515358361775e-06, "loss": 0.0178, "num_tokens": 46099527.0, "reward": 0.4000000059604645, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290207743644714, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.4165968894958496, "sampling/importance_sampling_ratio/mean": 0.37969687581062317, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.257540225982666, "sampling/sampling_logp_difference/mean": 0.013242505490779877, "step": 432, "step_time": 76.92222218425013 }, { "clip_ratio/high_max": 0.010825485485838726, "clip_ratio/high_mean": 0.0028439287998480722, "clip_ratio/low_mean": 0.0027954998950008303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005639428738504648, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 868.125, "completions/mean_terminated_length": 457.18182373046875, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.10676234681159258, "epoch": 0.24630261660978384, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.050377510488033295, "kl": 0.01319847971899435, "learning_rate": 6.78839590443686e-06, "loss": -0.1273, "num_tokens": 46210412.0, "reward": 0.4749999940395355, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.8954875469207764, "sampling/importance_sampling_ratio/mean": 0.4725480079650879, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1508989334106445, "sampling/sampling_logp_difference/mean": 0.01269255019724369, "step": 433, "step_time": 72.99516151030548 }, { "clip_ratio/high_max": 0.014108044619206339, "clip_ratio/high_mean": 0.00440389133291319, "clip_ratio/low_mean": 0.0013916015486756805, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005795492819743231, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 890.6500244140625, "completions/mean_terminated_length": 643.0, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "entropy": 0.108853823505342, "epoch": 0.24687144482366324, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02199239656329155, "kl": 0.01079841546015814, "learning_rate": 6.783276450511946e-06, "loss": -0.0016, "num_tokens": 46317046.0, "reward": 0.699999988079071, "reward_std": 0.9660917520523071, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.6016604900360107, "sampling/importance_sampling_ratio/mean": 0.5372885465621948, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1439695358276367, "sampling/sampling_logp_difference/mean": 0.012054559774696827, "step": 434, "step_time": 74.89235550747253 }, { "clip_ratio/high_max": 0.017226289492100477, "clip_ratio/high_mean": 0.006202723481692374, "clip_ratio/low_mean": 0.0014892578255967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007691981096286327, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 922.0, "completions/mean_length": 803.2750244140625, "completions/mean_terminated_length": 582.5499877929688, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.16267547756433487, "epoch": 0.24744027303754265, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02005886659026146, "kl": 0.019543458125554025, "learning_rate": 6.77815699658703e-06, "loss": -0.0633, "num_tokens": 46432473.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.7831475734710693, "sampling/importance_sampling_ratio/mean": 0.3139200806617737, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.035844564437866, "sampling/sampling_logp_difference/mean": 0.017081446945667267, "step": 435, "step_time": 77.03746780776419 }, { "clip_ratio/high_max": 0.010059168329462409, "clip_ratio/high_mean": 0.002363946317927912, "clip_ratio/low_mean": 0.0016221049736486748, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00398605118971318, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 846.0, "completions/mean_length": 836.9249877929688, "completions/mean_terminated_length": 583.8235473632812, "completions/min_length": 418.0, "completions/min_terminated_length": 418.0, "entropy": 0.08406325709074736, "epoch": 0.24800910125142206, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015598650090396404, "kl": 0.009656586567871273, "learning_rate": 6.773037542662116e-06, "loss": 0.0159, "num_tokens": 46536622.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.808912992477417, "sampling/importance_sampling_ratio/mean": 0.5265381336212158, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.880188465118408, "sampling/sampling_logp_difference/mean": 0.011099680326879025, "step": 436, "step_time": 73.07374306372367 }, { "clip_ratio/high_max": 0.018578588729724288, "clip_ratio/high_mean": 0.004586169234244153, "clip_ratio/low_mean": 0.001722880046145292, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00630904920399189, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 947.0, "completions/mean_length": 794.1749877929688, "completions/mean_terminated_length": 564.3500366210938, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.11888472270220518, "epoch": 0.24857792946530147, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.032251399010419846, "kl": 0.010213861707597971, "learning_rate": 6.767918088737202e-06, "loss": -0.0375, "num_tokens": 46639701.0, "reward": 0.75, "reward_std": 0.8697184920310974, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 1.9555420875549316, "sampling/importance_sampling_ratio/mean": 0.3801341652870178, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.876756191253662, "sampling/sampling_logp_difference/mean": 0.013563026674091816, "step": 437, "step_time": 73.6945853899233 }, { "clip_ratio/high_max": 0.010119033308001235, "clip_ratio/high_mean": 0.0025965339518734254, "clip_ratio/low_mean": 0.0025306469906354323, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005127181007992476, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 818.0, "completions/mean_length": 781.9500122070312, "completions/mean_terminated_length": 562.952392578125, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.10148964915424585, "epoch": 0.24914675767918087, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.05790752172470093, "kl": 0.010406493325717747, "learning_rate": 6.7627986348122874e-06, "loss": 0.0152, "num_tokens": 46732723.0, "reward": 0.925000011920929, "reward_std": 0.9442809820175171, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.695277214050293, "sampling/importance_sampling_ratio/mean": 0.46251097321510315, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.055673837661743, "sampling/sampling_logp_difference/mean": 0.013178231194615364, "step": 438, "step_time": 68.42209715326317 }, { "clip_ratio/high_max": 0.012294506072066724, "clip_ratio/high_mean": 0.0042353390890639275, "clip_ratio/low_mean": 0.002237453758425545, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006472792767453939, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 950.0, "completions/mean_length": 751.7750244140625, "completions/mean_terminated_length": 605.1923217773438, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.0807918393984437, "epoch": 0.24971558589306028, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02194744348526001, "kl": 0.014326289761811495, "learning_rate": 6.757679180887372e-06, "loss": 0.1712, "num_tokens": 46837026.0, "reward": 1.125, "reward_std": 0.9111137986183167, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.613369941711426, "sampling/importance_sampling_ratio/mean": 0.4689522683620453, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.591944694519043, "sampling/sampling_logp_difference/mean": 0.011806539259850979, "step": 439, "step_time": 73.19020007713698 }, { "clip_ratio/high_max": 0.013563027954660356, "clip_ratio/high_mean": 0.003486414934741333, "clip_ratio/low_mean": 0.0005126953019498615, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003999110253062099, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 825.375, "completions/mean_terminated_length": 626.75, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.15041633136570454, "epoch": 0.2502844141069397, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.02548333629965782, "kl": 0.01264808396808803, "learning_rate": 6.7525597269624575e-06, "loss": -0.0656, "num_tokens": 46950265.0, "reward": 0.875, "reward_std": 0.9388344883918762, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.605773687362671, "sampling/importance_sampling_ratio/mean": 0.5373753309249878, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7400554418563843, "sampling/sampling_logp_difference/mean": 0.01677497662603855, "step": 440, "step_time": 76.19398475415073 }, { "clip_ratio/high_max": 0.00999722653068602, "clip_ratio/high_mean": 0.0029959300882183015, "clip_ratio/low_mean": 0.004182976423180662, "clip_ratio/low_min": 0.0013084587408229709, "clip_ratio/region_mean": 0.007178906467743218, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 947.0, "completions/mean_length": 875.7000122070312, "completions/mean_terminated_length": 694.4444580078125, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.12464907579123974, "epoch": 0.2508532423208191, "frac_reward_zero_std": 0.0, "grad_norm": 0.023945238441228867, "kl": 0.011552821146324277, "learning_rate": 6.747440273037543e-06, "loss": -0.0908, "num_tokens": 47041789.0, "reward": 0.699999988079071, "reward_std": 0.8533493876457214, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.4363921880722046, "sampling/importance_sampling_ratio/mean": 0.260650098323822, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.797323703765869, "sampling/sampling_logp_difference/mean": 0.014443366788327694, "step": 441, "step_time": 71.94805638352409 }, { "clip_ratio/high_max": 0.012250804575160146, "clip_ratio/high_mean": 0.003831121706753038, "clip_ratio/low_mean": 0.0016601562674622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005491278017871082, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 875.7000122070312, "completions/mean_terminated_length": 567.6923217773438, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.07369831949472427, "epoch": 0.2514220705346985, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.21458937227725983, "kl": 0.038128197309561074, "learning_rate": 6.742320819112628e-06, "loss": -0.1071, "num_tokens": 47153025.0, "reward": 0.6000000238418579, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.5008697509765625, "sampling/importance_sampling_ratio/mean": 0.42329367995262146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.220969200134277, "sampling/sampling_logp_difference/mean": 0.011074538342654705, "step": 442, "step_time": 75.2896364305634 }, { "clip_ratio/high_max": 0.007961585477460176, "clip_ratio/high_mean": 0.0021802813425892964, "clip_ratio/low_mean": 0.0021240234200377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004304304820834659, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 787.0, "completions/mean_length": 930.2750244140625, "completions/mean_terminated_length": 555.375, "completions/min_length": 393.0, "completions/min_terminated_length": 393.0, "entropy": 0.10083794966340065, "epoch": 0.25199089874857794, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0510222502052784, "kl": 0.009787245595362037, "learning_rate": 6.737201365187713e-06, "loss": -0.1226, "num_tokens": 47259548.0, "reward": 0.375, "reward_std": 0.7741827964782715, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.757794141769409, "sampling/importance_sampling_ratio/mean": 0.5693306922912598, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.953416347503662, "sampling/sampling_logp_difference/mean": 0.01172153651714325, "step": 443, "step_time": 74.95088276802562 }, { "clip_ratio/high_max": 0.011025305837392807, "clip_ratio/high_mean": 0.003288728530606022, "clip_ratio/low_mean": 0.002148437488358468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005437166168121621, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 953.0, "completions/mean_length": 899.0250244140625, "completions/mean_terminated_length": 690.7333984375, "completions/min_length": 464.0, "completions/min_terminated_length": 464.0, "entropy": 0.09679889213293791, "epoch": 0.2525597269624573, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03980788588523865, "kl": 0.009394008433446288, "learning_rate": 6.732081911262798e-06, "loss": -0.0604, "num_tokens": 47362181.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.4514386653900146, "sampling/importance_sampling_ratio/mean": 0.3992556631565094, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.974778413772583, "sampling/sampling_logp_difference/mean": 0.01264180988073349, "step": 444, "step_time": 72.02996744122356 }, { "clip_ratio/high_max": 0.012672224547713995, "clip_ratio/high_mean": 0.004819347901502624, "clip_ratio/low_mean": 0.0013916015668655746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006210949562955648, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 818.0, "completions/mean_length": 804.6000366210938, "completions/mean_terminated_length": 536.4444580078125, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.09175539575517178, "epoch": 0.25312855517633676, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021733537316322327, "kl": 0.013216102146543562, "learning_rate": 6.726962457337885e-06, "loss": -0.0058, "num_tokens": 47474821.0, "reward": 0.9000000357627869, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.648782253265381, "sampling/importance_sampling_ratio/mean": 0.3528805375099182, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4325571060180664, "sampling/sampling_logp_difference/mean": 0.013016370125114918, "step": 445, "step_time": 75.1584263658151 }, { "clip_ratio/high_max": 0.015114778594579548, "clip_ratio/high_mean": 0.004599241525284015, "clip_ratio/low_mean": 0.00316110355197452, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007760345004498959, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 942.0, "completions/mean_length": 816.0250244140625, "completions/mean_terminated_length": 561.8333129882812, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.0975168664008379, "epoch": 0.25369738339021614, "frac_reward_zero_std": 0.0, "grad_norm": 0.019120968878269196, "kl": 0.012486472143791616, "learning_rate": 6.72184300341297e-06, "loss": 0.0523, "num_tokens": 47580950.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.3903528451919556, "sampling/importance_sampling_ratio/mean": 0.3464009761810303, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.222360372543335, "sampling/sampling_logp_difference/mean": 0.013292777352035046, "step": 446, "step_time": 73.60921168187633 }, { "clip_ratio/high_max": 0.010227321356069297, "clip_ratio/high_mean": 0.0032729864033171907, "clip_ratio/low_mean": 0.001635742220969405, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004908728617010638, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 904.0, "completions/mean_length": 656.5250244140625, "completions/mean_terminated_length": 499.0357360839844, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.13185467012226582, "epoch": 0.25426621160409557, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03918565437197685, "kl": 0.011697799316607416, "learning_rate": 6.716723549488055e-06, "loss": -0.1326, "num_tokens": 47679203.0, "reward": 1.399999976158142, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.699999988079071, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.7041051387786865, "sampling/importance_sampling_ratio/mean": 0.4347901940345764, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1180007457733154, "sampling/sampling_logp_difference/mean": 0.017039699479937553, "step": 447, "step_time": 69.55233595590107 }, { "clip_ratio/high_max": 0.010353048099204898, "clip_ratio/high_mean": 0.0025711574708111584, "clip_ratio/low_mean": 0.001928710931679234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004499868315178901, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 972.7250366210938, "completions/mean_terminated_length": 818.9000244140625, "completions/min_length": 544.0, "completions/min_terminated_length": 544.0, "entropy": 0.13233854714781046, "epoch": 0.25483503981797495, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04429463669657707, "kl": 0.010150517453439534, "learning_rate": 6.71160409556314e-06, "loss": -0.1367, "num_tokens": 47791656.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.0690667629241943, "sampling/importance_sampling_ratio/mean": 0.38573965430259705, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5951759815216064, "sampling/sampling_logp_difference/mean": 0.014633703045547009, "step": 448, "step_time": 77.08481032308191 }, { "clip_ratio/high_max": 0.010088010167237371, "clip_ratio/high_mean": 0.003021857774001546, "clip_ratio/low_mean": 0.00402463847422041, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.0070464962045662105, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 897.5499877929688, "completions/mean_terminated_length": 662.7142944335938, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.13938248343765736, "epoch": 0.2554038680318544, "frac_reward_zero_std": 0.0, "grad_norm": 0.04001111537218094, "kl": 0.015050576243083924, "learning_rate": 6.7064846416382255e-06, "loss": -0.0637, "num_tokens": 47908782.0, "reward": 0.574999988079071, "reward_std": 0.8439072966575623, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.8616812229156494, "sampling/importance_sampling_ratio/mean": 0.34326061606407166, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0436999797821045, "sampling/sampling_logp_difference/mean": 0.015265608206391335, "step": 449, "step_time": 77.97772746509872 }, { "clip_ratio/high_max": 0.010002174763940275, "clip_ratio/high_mean": 0.00404518761934014, "clip_ratio/low_mean": 0.00048828125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004533468891168013, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 806.5250244140625, "completions/mean_terminated_length": 566.1578979492188, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.06732895970344543, "epoch": 0.25597269624573377, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.00966737698763609, "kl": 0.010011613776441664, "learning_rate": 6.701365187713311e-06, "loss": 0.0049, "num_tokens": 48014219.0, "reward": 0.7750000357627869, "reward_std": 0.8911962509155273, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.4121909141540527, "sampling/importance_sampling_ratio/mean": 0.4161199629306793, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9860254526138306, "sampling/sampling_logp_difference/mean": 0.010212332010269165, "step": 450, "step_time": 73.41845967015252 }, { "clip_ratio/high_max": 0.014586741337552667, "clip_ratio/high_mean": 0.005177173225092702, "clip_ratio/low_mean": 0.0013764043178525753, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0065535776084288955, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 990.0, "completions/mean_length": 763.7750244140625, "completions/mean_terminated_length": 445.72222900390625, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.1102887075394392, "epoch": 0.2565415244596132, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018958086147904396, "kl": 0.02076984045561403, "learning_rate": 6.6962457337883955e-06, "loss": 0.0161, "num_tokens": 48122634.0, "reward": 0.8500000238418579, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.2870209217071533, "sampling/importance_sampling_ratio/mean": 0.32904601097106934, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.928138256072998, "sampling/sampling_logp_difference/mean": 0.014104440808296204, "step": 451, "step_time": 75.58497197693214 }, { "clip_ratio/high_max": 0.012738605902995914, "clip_ratio/high_mean": 0.003019989948370494, "clip_ratio/low_mean": 0.0013427734484139364, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00436276342952624, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 705.7250366210938, "completions/mean_terminated_length": 552.4815063476562, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.11829172912985086, "epoch": 0.2571103526734926, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.027457281947135925, "kl": 0.01569437189027667, "learning_rate": 6.691126279863481e-06, "loss": -0.0669, "num_tokens": 48225799.0, "reward": 1.350000023841858, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.9406943321228027, "sampling/importance_sampling_ratio/mean": 0.3352566659450531, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2224318981170654, "sampling/sampling_logp_difference/mean": 0.016090229153633118, "step": 452, "step_time": 71.5442235621158 }, { "clip_ratio/high_max": 0.012869176571257412, "clip_ratio/high_mean": 0.0034632282913662493, "clip_ratio/low_mean": 0.0006591796973225428, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00412240787409246, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 565.0, "completions/mean_length": 844.7000122070312, "completions/mean_terminated_length": 472.3077087402344, "completions/min_length": 403.0, "completions/min_terminated_length": 403.0, "entropy": 0.060489170253276825, "epoch": 0.257679180887372, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01049939263612032, "kl": 0.010512260079849511, "learning_rate": 6.686006825938567e-06, "loss": 0.0629, "num_tokens": 48330459.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.265073537826538, "sampling/importance_sampling_ratio/mean": 0.5453031659126282, "sampling/importance_sampling_ratio/min": 0.005950829479843378, "sampling/sampling_logp_difference/max": 2.703944206237793, "sampling/sampling_logp_difference/mean": 0.009725580923259258, "step": 453, "step_time": 72.22520999377593 }, { "clip_ratio/high_max": 0.01300062210066244, "clip_ratio/high_mean": 0.004778900882229209, "clip_ratio/low_mean": 0.0015288663307728712, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006307767209364101, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 945.0, "completions/mean_length": 591.7750244140625, "completions/mean_terminated_length": 483.71875, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.09674752317368984, "epoch": 0.2582480091012514, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04572293534874916, "kl": 0.014106053276918828, "learning_rate": 6.680887372013653e-06, "loss": 0.0887, "num_tokens": 48427178.0, "reward": 1.475000023841858, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.7115588188171387, "sampling/importance_sampling_ratio/mean": 0.48821887373924255, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6582071781158447, "sampling/sampling_logp_difference/mean": 0.014338771812617779, "step": 454, "step_time": 69.61492096190341 }, { "clip_ratio/high_max": 0.008346853399416432, "clip_ratio/high_mean": 0.001982581838092301, "clip_ratio/low_mean": 0.002026367194048362, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004008949006674811, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 868.0, "completions/mean_length": 947.125, "completions/mean_terminated_length": 716.5, "completions/min_length": 518.0, "completions/min_terminated_length": 518.0, "entropy": 0.0945558249950409, "epoch": 0.25881683731513083, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.035687536001205444, "kl": 0.011324930121190846, "learning_rate": 6.675767918088737e-06, "loss": -0.1007, "num_tokens": 48530351.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.8204433917999268, "sampling/importance_sampling_ratio/mean": 0.4319915771484375, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.034660339355469, "sampling/sampling_logp_difference/mean": 0.012226960621774197, "step": 455, "step_time": 71.86996091390029 }, { "clip_ratio/high_max": 0.014534920454025269, "clip_ratio/high_mean": 0.005112173399538733, "clip_ratio/low_mean": 0.002197265643189894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00730943912640214, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 966.0, "completions/mean_length": 735.6749877929688, "completions/mean_terminated_length": 562.6799926757812, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "entropy": 0.10638454556465149, "epoch": 0.2593856655290102, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.012117978185415268, "kl": 0.01459362602327019, "learning_rate": 6.670648464163823e-06, "loss": -0.0036, "num_tokens": 48625850.0, "reward": 1.2000000476837158, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.384063959121704, "sampling/importance_sampling_ratio/mean": 0.3467779755592346, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.593806505203247, "sampling/sampling_logp_difference/mean": 0.01490786112844944, "step": 456, "step_time": 71.97612849110737 }, { "clip_ratio/high_max": 0.012993267038837075, "clip_ratio/high_mean": 0.003663230047095567, "clip_ratio/low_mean": 0.0018066406228172127, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005469870782690123, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 597.0, "completions/mean_length": 868.3500366210938, "completions/mean_terminated_length": 401.3999938964844, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.07134720264002681, "epoch": 0.25995449374288965, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03236772119998932, "kl": 0.015024264226667583, "learning_rate": 6.665529010238908e-06, "loss": -0.0274, "num_tokens": 48729408.0, "reward": 0.42500001192092896, "reward_std": 0.780778706073761, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.2803497314453125, "sampling/importance_sampling_ratio/mean": 0.2714594900608063, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4141464233398438, "sampling/sampling_logp_difference/mean": 0.009978722780942917, "step": 457, "step_time": 72.32496971613728 }, { "clip_ratio/high_max": 0.008040659478865564, "clip_ratio/high_mean": 0.0021919002683716826, "clip_ratio/low_mean": 0.000561523440410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0027534237160580233, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.0, "completions/mean_length": 848.4500122070312, "completions/mean_terminated_length": 633.888916015625, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.09032941237092018, "epoch": 0.2605233219567691, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.010846825316548347, "kl": 0.013603603350929916, "learning_rate": 6.6604095563139936e-06, "loss": -0.0213, "num_tokens": 48839210.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.464590311050415, "sampling/importance_sampling_ratio/mean": 0.3505876064300537, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4556052684783936, "sampling/sampling_logp_difference/mean": 0.013366976752877235, "step": 458, "step_time": 74.79452346474864 }, { "clip_ratio/high_max": 0.015048189321532845, "clip_ratio/high_mean": 0.005446772265713662, "clip_ratio/low_mean": 0.0012207031086290954, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006667475448921323, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 978.0, "completions/mean_length": 837.1000366210938, "completions/mean_terminated_length": 668.0, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.10585079900920391, "epoch": 0.26109215017064846, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0627639889717102, "kl": 0.011049693683162332, "learning_rate": 6.655290102389078e-06, "loss": 0.2027, "num_tokens": 48946758.0, "reward": 0.925000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.7875497341156006, "sampling/importance_sampling_ratio/mean": 0.4632607400417328, "sampling/importance_sampling_ratio/min": 0.008729131892323494, "sampling/sampling_logp_difference/max": 1.9108437299728394, "sampling/sampling_logp_difference/mean": 0.012937471270561218, "step": 459, "step_time": 74.94861962785944 }, { "clip_ratio/high_max": 0.012187159096356481, "clip_ratio/high_mean": 0.004873907659202814, "clip_ratio/low_mean": 0.0007568359360448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00563074357341975, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 815.5, "completions/mean_terminated_length": 644.9091186523438, "completions/min_length": 423.0, "completions/min_terminated_length": 423.0, "entropy": 0.05969715537503362, "epoch": 0.2616609783845279, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.012592175975441933, "kl": 0.010648412629961967, "learning_rate": 6.650170648464164e-06, "loss": 0.0214, "num_tokens": 49049434.0, "reward": 0.9750000238418579, "reward_std": 0.9469925761222839, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.953073263168335, "sampling/importance_sampling_ratio/mean": 0.3238994777202606, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.560039758682251, "sampling/sampling_logp_difference/mean": 0.010052088648080826, "step": 460, "step_time": 73.70498062926345 }, { "clip_ratio/high_max": 0.012274422100745142, "clip_ratio/high_mean": 0.00402566758566536, "clip_ratio/low_mean": 0.0032476075284648687, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007273275114130229, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 875.9000244140625, "completions/mean_terminated_length": 600.857177734375, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "entropy": 0.1101917615160346, "epoch": 0.2622298065984073, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022503523156046867, "kl": 0.02276251488365233, "learning_rate": 6.64505119453925e-06, "loss": -0.1078, "num_tokens": 49163054.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.6646149158477783, "sampling/importance_sampling_ratio/mean": 0.40720272064208984, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.738414764404297, "sampling/sampling_logp_difference/mean": 0.015791192650794983, "step": 461, "step_time": 76.05464000348002 }, { "clip_ratio/high_max": 0.017482523107901216, "clip_ratio/high_mean": 0.006276576605159789, "clip_ratio/low_mean": 0.0017146395912277512, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00799121632007882, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 881.0, "completions/mean_length": 838.5250244140625, "completions/mean_terminated_length": 653.0499877929688, "completions/min_length": 476.0, "completions/min_terminated_length": 476.0, "entropy": 0.12517081759870052, "epoch": 0.2627986348122867, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.038889795541763306, "kl": 0.016759993508458138, "learning_rate": 6.6399317406143345e-06, "loss": 0.0525, "num_tokens": 49276427.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.6083390712738037, "sampling/importance_sampling_ratio/mean": 0.27646762132644653, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7860747575759888, "sampling/sampling_logp_difference/mean": 0.016493311151862144, "step": 462, "step_time": 76.13138928334229 }, { "clip_ratio/high_max": 0.01411367062246427, "clip_ratio/high_mean": 0.005882266414118931, "clip_ratio/low_mean": 0.0009277343797293724, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006810000719269738, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 765.6500244140625, "completions/mean_terminated_length": 531.90478515625, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.10027127619832754, "epoch": 0.2633674630261661, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022125136107206345, "kl": 0.01608140196185559, "learning_rate": 6.63481228668942e-06, "loss": -0.031, "num_tokens": 49375677.0, "reward": 1.0500000715255737, "reward_std": 1.0114725828170776, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.574547529220581, "sampling/importance_sampling_ratio/mean": 0.3770771026611328, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8096325397491455, "sampling/sampling_logp_difference/mean": 0.014104780741035938, "step": 463, "step_time": 71.47289959294721 }, { "clip_ratio/high_max": 0.013283459993544966, "clip_ratio/high_mean": 0.004555096180411056, "clip_ratio/low_mean": 0.0032470703081344254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007802166452165693, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 904.1000366210938, "completions/mean_terminated_length": 704.2667236328125, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.123308340087533, "epoch": 0.26393629124004553, "frac_reward_zero_std": 0.0, "grad_norm": 0.027579771354794502, "kl": 0.012785089667886496, "learning_rate": 6.629692832764505e-06, "loss": 0.2004, "num_tokens": 49478825.0, "reward": 0.574999988079071, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.9619781970977783, "sampling/importance_sampling_ratio/mean": 0.4976072311401367, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.148780584335327, "sampling/sampling_logp_difference/mean": 0.01530677080154419, "step": 464, "step_time": 74.08839026838541 }, { "clip_ratio/high_max": 0.013428260921500623, "clip_ratio/high_mean": 0.004107974556973204, "clip_ratio/low_mean": 0.002368164074141532, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0064761384564917535, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 892.5499877929688, "completions/mean_terminated_length": 585.8333740234375, "completions/min_length": 386.0, "completions/min_terminated_length": 386.0, "entropy": 0.13003280758857727, "epoch": 0.2645051194539249, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01692988909780979, "kl": 0.015507351257838309, "learning_rate": 6.624573378839591e-06, "loss": 0.1475, "num_tokens": 49587431.0, "reward": 0.5, "reward_std": 0.8164966106414795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.9918566942214966, "sampling/importance_sampling_ratio/mean": 0.5607815980911255, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9471356868743896, "sampling/sampling_logp_difference/mean": 0.015807554125785828, "step": 465, "step_time": 73.16075699776411 }, { "clip_ratio/high_max": 0.008271901635453105, "clip_ratio/high_mean": 0.001970836066902848, "clip_ratio/low_mean": 0.002148437466530595, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004119273478863761, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 944.9249877929688, "completions/mean_terminated_length": 572.1428833007812, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.11512178927659988, "epoch": 0.26507394766780434, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.034874558448791504, "kl": 0.037188217509537935, "learning_rate": 6.619453924914675e-06, "loss": -0.0307, "num_tokens": 49688484.0, "reward": 0.3499999940395355, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.0035977363586426, "sampling/importance_sampling_ratio/mean": 0.3493315875530243, "sampling/importance_sampling_ratio/min": 0.0013698587426915765, "sampling/sampling_logp_difference/max": 3.6919450759887695, "sampling/sampling_logp_difference/mean": 0.013691926375031471, "step": 466, "step_time": 72.12309839460067 }, { "clip_ratio/high_max": 0.01329484541201964, "clip_ratio/high_mean": 0.0039432143967133015, "clip_ratio/low_mean": 0.0010466427993378602, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0049898571451194584, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 725.8250122070312, "completions/mean_terminated_length": 527.0416870117188, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.08998007420450449, "epoch": 0.2656427758816837, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.023376302793622017, "kl": 0.031904300791211426, "learning_rate": 6.614334470989761e-06, "loss": 0.0321, "num_tokens": 49789949.0, "reward": 1.1750000715255737, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.098562002182007, "sampling/importance_sampling_ratio/mean": 0.3818635940551758, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0536117553710938, "sampling/sampling_logp_difference/mean": 0.013894395902752876, "step": 467, "step_time": 78.72964439168572 }, { "clip_ratio/high_max": 0.012085718219168484, "clip_ratio/high_mean": 0.0035541510733310133, "clip_ratio/low_mean": 0.001440429699869128, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004994580609491095, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 878.7750244140625, "completions/mean_terminated_length": 577.1538696289062, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.10261795297265053, "epoch": 0.26621160409556316, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01633189059793949, "kl": 0.012621246161870658, "learning_rate": 6.609215017064847e-06, "loss": 0.054, "num_tokens": 49890260.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.7447198629379272, "sampling/importance_sampling_ratio/mean": 0.3155122399330139, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6131412982940674, "sampling/sampling_logp_difference/mean": 0.01342795044183731, "step": 468, "step_time": 70.3689693317283 }, { "clip_ratio/high_max": 0.01586463482817635, "clip_ratio/high_mean": 0.006221352727152407, "clip_ratio/low_mean": 0.0013916015741415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007612954301293939, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 753.625, "completions/mean_terminated_length": 532.4091186523438, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.10537755582481623, "epoch": 0.26678043230944254, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023871110752224922, "kl": 0.016701795044355094, "learning_rate": 6.6040955631399325e-06, "loss": -0.0068, "num_tokens": 49993437.0, "reward": 0.824999988079071, "reward_std": 0.8439072966575623, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.677431106567383, "sampling/importance_sampling_ratio/mean": 0.5141145586967468, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.4297115802764893, "sampling/sampling_logp_difference/mean": 0.014981674961745739, "step": 469, "step_time": 70.78423190489411 }, { "clip_ratio/high_max": 0.014946457697078586, "clip_ratio/high_mean": 0.0038005895912647247, "clip_ratio/low_mean": 0.0029052734025754035, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006705862993840128, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 936.0, "completions/mean_terminated_length": 672.0, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.13439391646534204, "epoch": 0.267349260523322, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03493208810687065, "kl": 0.013632197631523013, "learning_rate": 6.598976109215017e-06, "loss": -0.126, "num_tokens": 50105541.0, "reward": 0.45000001788139343, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.8295774459838867, "sampling/importance_sampling_ratio/mean": 0.319611519575119, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9810872077941895, "sampling/sampling_logp_difference/mean": 0.015814509242773056, "step": 470, "step_time": 74.99740229197778 }, { "clip_ratio/high_max": 0.013283913722261786, "clip_ratio/high_mean": 0.004007622716017067, "clip_ratio/low_mean": 0.0015869140843278728, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005594536894932389, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 938.0, "completions/mean_length": 793.875, "completions/mean_terminated_length": 410.3333435058594, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.10723326727747917, "epoch": 0.26791808873720135, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01706683449447155, "kl": 0.01165473903529346, "learning_rate": 6.5938566552901025e-06, "loss": 0.0343, "num_tokens": 50204768.0, "reward": 0.625, "reward_std": 0.8678739070892334, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.2878241539001465, "sampling/importance_sampling_ratio/mean": 0.42863669991493225, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7063082456588745, "sampling/sampling_logp_difference/mean": 0.013418108224868774, "step": 471, "step_time": 70.93291475786828 }, { "clip_ratio/high_max": 0.011875801021233201, "clip_ratio/high_mean": 0.003059023089008406, "clip_ratio/low_mean": 0.00141601563700533, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004475038644159213, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 878.0, "completions/mean_length": 780.6749877929688, "completions/mean_terminated_length": 600.8261108398438, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.07454119250178337, "epoch": 0.2684869169510808, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.016290828585624695, "kl": 0.012414090742822737, "learning_rate": 6.588737201365188e-06, "loss": 0.0462, "num_tokens": 50291491.0, "reward": 1.125, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.9527041912078857, "sampling/importance_sampling_ratio/mean": 0.4650319218635559, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7743914127349854, "sampling/sampling_logp_difference/mean": 0.0104582030326128, "step": 472, "step_time": 72.3140124573838 }, { "clip_ratio/high_max": 0.018139236606657505, "clip_ratio/high_mean": 0.007179496577009559, "clip_ratio/low_mean": 0.0011230468953726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008302543545141816, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 854.0, "completions/mean_length": 772.2000122070312, "completions/mean_terminated_length": 544.3809814453125, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.07357210014015436, "epoch": 0.26905574516496017, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04627251252532005, "kl": 0.014018231420777738, "learning_rate": 6.583617747440273e-06, "loss": 0.1091, "num_tokens": 50397851.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057363510131836, "sampling/importance_sampling_ratio/max": 1.9477766752243042, "sampling/importance_sampling_ratio/mean": 0.34647852182388306, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1070849895477295, "sampling/sampling_logp_difference/mean": 0.0122078787535429, "step": 473, "step_time": 73.57376698986627 }, { "clip_ratio/high_max": 0.006858390581328422, "clip_ratio/high_mean": 0.0015137236041482538, "clip_ratio/low_mean": 0.0019557976702344604, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034695212671067566, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 798.0, "completions/mean_length": 930.9000244140625, "completions/mean_terminated_length": 558.5, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.0855887122452259, "epoch": 0.2696245733788396, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.02587939240038395, "kl": 0.01055869273841381, "learning_rate": 6.578498293515358e-06, "loss": -0.0568, "num_tokens": 50512551.0, "reward": 0.32500001788139343, "reward_std": 0.6938373446464539, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.408355236053467, "sampling/importance_sampling_ratio/mean": 0.47746196389198303, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0167696475982666, "sampling/sampling_logp_difference/mean": 0.01203425694257021, "step": 474, "step_time": 77.36439739214256 }, { "clip_ratio/high_max": 0.012670263648033142, "clip_ratio/high_mean": 0.004305000999011099, "clip_ratio/low_mean": 0.003613281238358468, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.007918282295577228, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 910.875, "completions/mean_terminated_length": 675.923095703125, "completions/min_length": 433.0, "completions/min_terminated_length": 433.0, "entropy": 0.11169054545462132, "epoch": 0.270193401592719, "frac_reward_zero_std": 0.0, "grad_norm": 0.029268737882375717, "kl": 0.011096911272034049, "learning_rate": 6.573378839590443e-06, "loss": 0.1449, "num_tokens": 50628746.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.7101731300354004, "sampling/importance_sampling_ratio/mean": 0.4325999319553375, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.000739812850952, "sampling/sampling_logp_difference/mean": 0.013745561242103577, "step": 475, "step_time": 78.77858569519594 }, { "clip_ratio/high_max": 0.01507108984515071, "clip_ratio/high_mean": 0.00616612026351504, "clip_ratio/low_mean": 0.002319335955689894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008485456113703549, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 900.0, "completions/mean_terminated_length": 808.3478393554688, "completions/min_length": 569.0, "completions/min_terminated_length": 569.0, "entropy": 0.13898626808077097, "epoch": 0.2707622298065984, "frac_reward_zero_std": 0.0, "grad_norm": 0.024247560650110245, "kl": 0.012397846905514598, "learning_rate": 6.56825938566553e-06, "loss": 0.0815, "num_tokens": 50734330.0, "reward": 0.949999988079071, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.5327749252319336, "sampling/importance_sampling_ratio/mean": 0.30790895223617554, "sampling/importance_sampling_ratio/min": 0.000946055690292269, "sampling/sampling_logp_difference/max": 3.1504592895507812, "sampling/sampling_logp_difference/mean": 0.016303934156894684, "step": 476, "step_time": 74.08598350244574 }, { "clip_ratio/high_max": 0.01191171124810353, "clip_ratio/high_mean": 0.00471447165182326, "clip_ratio/low_mean": 0.0023193359447759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0070338076911866665, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 853.8250122070312, "completions/mean_terminated_length": 645.8333129882812, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.12275784555822611, "epoch": 0.2713310580204778, "frac_reward_zero_std": 0.0, "grad_norm": 0.022561002522706985, "kl": 0.016277934424579144, "learning_rate": 6.563139931740615e-06, "loss": 0.0407, "num_tokens": 50836211.0, "reward": 0.8500000238418579, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.5288546085357666, "sampling/importance_sampling_ratio/mean": 0.2980157732963562, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.107971668243408, "sampling/sampling_logp_difference/mean": 0.014239159412682056, "step": 477, "step_time": 71.71365914493799 }, { "clip_ratio/high_max": 0.006508045451482758, "clip_ratio/high_mean": 0.001967267169675324, "clip_ratio/low_mean": 0.002392578113358468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004359845202998258, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 974.1000366210938, "completions/mean_terminated_length": 774.5, "completions/min_length": 656.0, "completions/min_terminated_length": 656.0, "entropy": 0.09282228723168373, "epoch": 0.27189988623435724, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014694984070956707, "kl": 0.009664640121627599, "learning_rate": 6.5580204778157e-06, "loss": -0.0181, "num_tokens": 50945839.0, "reward": 0.4000000059604645, "reward_std": 0.8101914525032043, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.7236268520355225, "sampling/importance_sampling_ratio/mean": 0.28273525834083557, "sampling/importance_sampling_ratio/min": 0.0007980837835930288, "sampling/sampling_logp_difference/max": 5.777853012084961, "sampling/sampling_logp_difference/mean": 0.011824283748865128, "step": 478, "step_time": 76.27796727581881 }, { "clip_ratio/high_max": 0.014318058441858739, "clip_ratio/high_mean": 0.004437030307599343, "clip_ratio/low_mean": 0.003964111892855726, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008401142375078052, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 885.5, "completions/mean_terminated_length": 677.75, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.14465549774467945, "epoch": 0.2724687144482366, "frac_reward_zero_std": 0.0, "grad_norm": 0.017596371471881866, "kl": 0.01262668869458139, "learning_rate": 6.552901023890785e-06, "loss": -0.0055, "num_tokens": 51057339.0, "reward": 0.699999988079071, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.9413046836853027, "sampling/importance_sampling_ratio/mean": 0.2693205773830414, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.741203784942627, "sampling/sampling_logp_difference/mean": 0.01611946150660515, "step": 479, "step_time": 75.0798760089092 }, { "clip_ratio/high_max": 0.007228973787277937, "clip_ratio/high_mean": 0.0022757344268029556, "clip_ratio/low_mean": 0.001708984384094947, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003984718816354871, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 891.0, "completions/mean_length": 961.9249877929688, "completions/mean_terminated_length": 713.625, "completions/min_length": 505.0, "completions/min_terminated_length": 505.0, "entropy": 0.09747061505913734, "epoch": 0.27303754266211605, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011226605623960495, "kl": 0.008613229962065816, "learning_rate": 6.547781569965871e-06, "loss": 0.0046, "num_tokens": 51168584.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.3727836608886719, "sampling/importance_sampling_ratio/mean": 0.29293292760849, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9409140348434448, "sampling/sampling_logp_difference/mean": 0.011934489011764526, "step": 480, "step_time": 75.11262708273716 }, { "clip_ratio/high_max": 0.004122735350392759, "clip_ratio/high_mean": 0.0011652178800432011, "clip_ratio/low_mean": 0.001391601559589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0025568194469087757, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 954.0, "completions/mean_length": 1012.7750244140625, "completions/mean_terminated_length": 911.75, "completions/min_length": 853.0, "completions/min_terminated_length": 853.0, "entropy": 0.1575841922312975, "epoch": 0.27360637087599543, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.029208069667220116, "kl": 0.009852639283053577, "learning_rate": 6.542662116040956e-06, "loss": -0.1187, "num_tokens": 51278927.0, "reward": 0.20000000298023224, "reward_std": 0.6076436042785645, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.5504186153411865, "sampling/importance_sampling_ratio/mean": 0.2643217444419861, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0156328678131104, "sampling/sampling_logp_difference/mean": 0.015465909615159035, "step": 481, "step_time": 74.53681279928423 }, { "clip_ratio/high_max": 0.01185701129725203, "clip_ratio/high_mean": 0.005069431186711881, "clip_ratio/low_mean": 0.0021728515857830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00724228291073814, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 830.6500244140625, "completions/mean_terminated_length": 637.2999877929688, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "entropy": 0.09698916226625443, "epoch": 0.27417519908987487, "frac_reward_zero_std": 0.0, "grad_norm": 0.030155375599861145, "kl": 0.01380425866227597, "learning_rate": 6.537542662116041e-06, "loss": 0.0415, "num_tokens": 51388529.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.83217191696167, "sampling/importance_sampling_ratio/mean": 0.45840054750442505, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4587390422821045, "sampling/sampling_logp_difference/mean": 0.012345574796199799, "step": 482, "step_time": 74.60369073483162 }, { "clip_ratio/high_max": 0.014407092414330691, "clip_ratio/high_mean": 0.004199211412924342, "clip_ratio/low_mean": 0.0016113281235448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005810539587400854, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 900.7999877929688, "completions/mean_terminated_length": 695.4666748046875, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.0778953991830349, "epoch": 0.27474402730375425, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029829971492290497, "kl": 0.009488232783041894, "learning_rate": 6.532423208191126e-06, "loss": -0.1043, "num_tokens": 51494745.0, "reward": 0.625, "reward_std": 0.8678739070892334, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.024505138397217, "sampling/importance_sampling_ratio/mean": 0.5890129804611206, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9739315509796143, "sampling/sampling_logp_difference/mean": 0.010589884594082832, "step": 483, "step_time": 75.27839852194302 }, { "clip_ratio/high_max": 0.007845864747650921, "clip_ratio/high_mean": 0.001734008124913089, "clip_ratio/low_mean": 0.0016113281162688509, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033453362120781094, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 610.0, "completions/mean_length": 931.0750122070312, "completions/mean_terminated_length": 493.0000305175781, "completions/min_length": 403.0, "completions/min_terminated_length": 403.0, "entropy": 0.0755372610874474, "epoch": 0.2753128555176337, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01944330520927906, "kl": 0.007443137466907501, "learning_rate": 6.527303754266212e-06, "loss": -0.0121, "num_tokens": 51601556.0, "reward": 0.3499999940395355, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.5015244483947754, "sampling/importance_sampling_ratio/mean": 0.3720167875289917, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.408406972885132, "sampling/sampling_logp_difference/mean": 0.009926686994731426, "step": 484, "step_time": 74.46381616033614 }, { "clip_ratio/high_max": 0.013866138295270503, "clip_ratio/high_mean": 0.003757886021048762, "clip_ratio/low_mean": 0.002783203119179234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006541089009260759, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 961.0, "completions/mean_length": 814.75, "completions/mean_terminated_length": 583.4736938476562, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.1347870547324419, "epoch": 0.27588168373151306, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023843159899115562, "kl": 0.010575480235274881, "learning_rate": 6.522184300341298e-06, "loss": 0.0748, "num_tokens": 51712010.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.8364415168762207, "sampling/importance_sampling_ratio/mean": 0.5192568302154541, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3005160093307495, "sampling/sampling_logp_difference/mean": 0.014614480547606945, "step": 485, "step_time": 75.59995581861585 }, { "clip_ratio/high_max": 0.01561996087548323, "clip_ratio/high_mean": 0.005598834482952952, "clip_ratio/low_mean": 0.0015815173246664926, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007180351589340717, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 718.4249877929688, "completions/mean_terminated_length": 571.2963256835938, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.09998158551752567, "epoch": 0.2764505119453925, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0346100740134716, "kl": 0.01276671577943489, "learning_rate": 6.517064846416382e-06, "loss": 0.0912, "num_tokens": 51818147.0, "reward": 1.3000000715255737, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.5138227939605713, "sampling/importance_sampling_ratio/mean": 0.5703026056289673, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8820314407348633, "sampling/sampling_logp_difference/mean": 0.013594272546470165, "step": 486, "step_time": 71.92991583910771 }, { "clip_ratio/high_max": 0.014616126660257578, "clip_ratio/high_mean": 0.005318716415786184, "clip_ratio/low_mean": 0.0008754310802032705, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006194147455971688, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 780.6749877929688, "completions/mean_terminated_length": 511.7368469238281, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.1755203828215599, "epoch": 0.2770193401592719, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024116849526762962, "kl": 0.014367373660206795, "learning_rate": 6.511945392491468e-06, "loss": -0.0627, "num_tokens": 51918462.0, "reward": 0.9000000357627869, "reward_std": 0.9818872213363647, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.6295254230499268, "sampling/importance_sampling_ratio/mean": 0.38261717557907104, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.726381778717041, "sampling/sampling_logp_difference/mean": 0.01817764900624752, "step": 487, "step_time": 78.87726871832274 }, { "clip_ratio/high_max": 0.01607350492849946, "clip_ratio/high_mean": 0.0047205940063577145, "clip_ratio/low_mean": 0.0015625000160071068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062830940005369484, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 905.9000244140625, "completions/mean_terminated_length": 709.0667114257812, "completions/min_length": 480.0, "completions/min_terminated_length": 480.0, "entropy": 0.11546470783650875, "epoch": 0.2775881683731513, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02758985385298729, "kl": 0.011140377493575215, "learning_rate": 6.506825938566553e-06, "loss": -0.0638, "num_tokens": 52023730.0, "reward": 0.625, "reward_std": 0.8678739666938782, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.820240020751953, "sampling/importance_sampling_ratio/mean": 0.40011897683143616, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.889596939086914, "sampling/sampling_logp_difference/mean": 0.01377650536596775, "step": 488, "step_time": 71.9637774170842 }, { "clip_ratio/high_max": 0.010099311708472669, "clip_ratio/high_mean": 0.003028576698852703, "clip_ratio/low_mean": 0.0034179687354480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0064465454197488725, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 949.0, "completions/mean_length": 911.9249877929688, "completions/mean_terminated_length": 650.4166870117188, "completions/min_length": 495.0, "completions/min_terminated_length": 495.0, "entropy": 0.13371362071484327, "epoch": 0.2781569965870307, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030694210901856422, "kl": 0.011676165508106351, "learning_rate": 6.501706484641639e-06, "loss": -0.0751, "num_tokens": 52130863.0, "reward": 0.6000000238418579, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.491182565689087, "sampling/importance_sampling_ratio/mean": 0.3865533769130707, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.473911762237549, "sampling/sampling_logp_difference/mean": 0.015549766831099987, "step": 489, "step_time": 72.63982973620296 }, { "clip_ratio/high_max": 0.010842481977306306, "clip_ratio/high_mean": 0.002665453081135638, "clip_ratio/low_mean": 0.00251464844950533, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005180101579753682, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 957.0750122070312, "completions/mean_terminated_length": 726.5555419921875, "completions/min_length": 526.0, "completions/min_terminated_length": 526.0, "entropy": 0.12242397852241993, "epoch": 0.2787258248009101, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0340958796441555, "kl": 0.009695721266325563, "learning_rate": 6.496587030716723e-06, "loss": -0.1195, "num_tokens": 52239578.0, "reward": 0.4000000059604645, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.490457057952881, "sampling/importance_sampling_ratio/mean": 0.40794000029563904, "sampling/importance_sampling_ratio/min": 0.0003250609152019024, "sampling/sampling_logp_difference/max": 1.7742674350738525, "sampling/sampling_logp_difference/mean": 0.013005462475121021, "step": 490, "step_time": 74.6088167114649 }, { "clip_ratio/high_max": 0.018762102350592613, "clip_ratio/high_mean": 0.005289000982884318, "clip_ratio/low_mean": 0.002709960979700554, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007998962071724236, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 963.0, "completions/mean_length": 873.0250244140625, "completions/mean_terminated_length": 520.75, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.14274496212601662, "epoch": 0.2792946530147895, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0335356630384922, "kl": 0.012638145126402378, "learning_rate": 6.491467576791809e-06, "loss": -0.0564, "num_tokens": 52353411.0, "reward": 0.6000000238418579, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.325406312942505, "sampling/importance_sampling_ratio/mean": 0.40122947096824646, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0269455909729004, "sampling/sampling_logp_difference/mean": 0.015436890535056591, "step": 491, "step_time": 75.03240982675925 }, { "clip_ratio/high_max": 0.010566328186541796, "clip_ratio/high_mean": 0.0023855968174757436, "clip_ratio/low_mean": 0.0005126953201397555, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028982921212445945, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 926.7750244140625, "completions/mean_terminated_length": 537.875, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "entropy": 0.055088881868869066, "epoch": 0.27986348122866894, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011449187994003296, "kl": 0.008369691611733288, "learning_rate": 6.486348122866895e-06, "loss": 0.0026, "num_tokens": 52466082.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.1006650924682617, "sampling/importance_sampling_ratio/mean": 0.4016456604003906, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.950954556465149, "sampling/sampling_logp_difference/mean": 0.007985718548297882, "step": 492, "step_time": 75.09792012465186 }, { "clip_ratio/high_max": 0.011040152981877327, "clip_ratio/high_mean": 0.0033221025514649227, "clip_ratio/low_mean": 0.0017578124861756805, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005079915019450709, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 971.0, "completions/mean_length": 864.7000122070312, "completions/mean_terminated_length": 533.84619140625, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.062106819823384285, "epoch": 0.2804323094425484, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02223166637122631, "kl": 0.01625496707856655, "learning_rate": 6.48122866894198e-06, "loss": -0.0335, "num_tokens": 52577342.0, "reward": 0.6500000357627869, "reward_std": 0.9486833214759827, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.386915683746338, "sampling/importance_sampling_ratio/mean": 0.35149139165878296, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.615715980529785, "sampling/sampling_logp_difference/mean": 0.009399927221238613, "step": 493, "step_time": 74.68729288689792 }, { "clip_ratio/high_max": 0.015405582031235099, "clip_ratio/high_mean": 0.005194740690058097, "clip_ratio/low_mean": 0.0020019531511934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007196693652076647, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 737.0750122070312, "completions/mean_terminated_length": 525.0, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.1016045855358243, "epoch": 0.28100113765642776, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03547416999936104, "kl": 0.019024177570827305, "learning_rate": 6.476109215017065e-06, "loss": 0.0088, "num_tokens": 52682585.0, "reward": 1.0500000715255737, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.492065906524658, "sampling/importance_sampling_ratio/mean": 0.35980886220932007, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8284645080566406, "sampling/sampling_logp_difference/mean": 0.014869351871311665, "step": 494, "step_time": 72.49810789129697 }, { "clip_ratio/high_max": 0.014988974784500897, "clip_ratio/high_mean": 0.004181737778708339, "clip_ratio/low_mean": 0.0014404297107830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005622167547699064, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 784.3250122070312, "completions/mean_terminated_length": 607.1739501953125, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.09271588735282421, "epoch": 0.2815699658703072, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015976514667272568, "kl": 0.012304963252972811, "learning_rate": 6.47098976109215e-06, "loss": 0.0752, "num_tokens": 52779806.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.3937883377075195, "sampling/importance_sampling_ratio/mean": 0.3420751690864563, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5062692165374756, "sampling/sampling_logp_difference/mean": 0.011677282862365246, "step": 495, "step_time": 71.81028197240084 }, { "clip_ratio/high_max": 0.010370773350587115, "clip_ratio/high_mean": 0.0030169449019012973, "clip_ratio/low_mean": 0.00019531250291038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003212257390259765, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 623.4750366210938, "completions/mean_terminated_length": 451.8214416503906, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.09817078709602356, "epoch": 0.2821387940841866, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009884404949843884, "kl": 0.01195009530056268, "learning_rate": 6.465870307167236e-06, "loss": -0.0321, "num_tokens": 52881425.0, "reward": 1.399999976158142, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.699999988079071, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.6101775169372559, "sampling/importance_sampling_ratio/mean": 0.4098493456840515, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.773529052734375, "sampling/sampling_logp_difference/mean": 0.014648260548710823, "step": 496, "step_time": 69.76004679175094 }, { "clip_ratio/high_max": 0.018063795287162066, "clip_ratio/high_mean": 0.007654806686332449, "clip_ratio/low_mean": 0.0012207031195430318, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00887550989864394, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 703.4249877929688, "completions/mean_terminated_length": 610.3547973632812, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.14410999976098537, "epoch": 0.282707622298066, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02567712776362896, "kl": 0.012453289236873388, "learning_rate": 6.46075085324232e-06, "loss": -0.0454, "num_tokens": 52980114.0, "reward": 1.350000023841858, "reward_std": 0.8335896730422974, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.8983805179595947, "sampling/importance_sampling_ratio/mean": 0.38441330194473267, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.012673854827881, "sampling/sampling_logp_difference/mean": 0.01661505363881588, "step": 497, "step_time": 73.07353889802471 }, { "clip_ratio/high_max": 0.011180753586813807, "clip_ratio/high_mean": 0.0032318926823791116, "clip_ratio/low_mean": 0.0027504804311320186, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005982373142614961, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 873.3500366210938, "completions/mean_terminated_length": 647.375, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.13369132671505213, "epoch": 0.2832764505119454, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.6794224381446838, "kl": 0.05165607831440866, "learning_rate": 6.455631399317406e-06, "loss": 0.0381, "num_tokens": 53090472.0, "reward": 0.699999988079071, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.222813129425049, "sampling/importance_sampling_ratio/mean": 0.27117133140563965, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.047417402267456, "sampling/sampling_logp_difference/mean": 0.015051759779453278, "step": 498, "step_time": 76.99801982729696 }, { "clip_ratio/high_max": 0.011547992995474488, "clip_ratio/high_mean": 0.003340006383950822, "clip_ratio/low_mean": 0.0028418138390406966, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006181820295751095, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 963.0, "completions/mean_length": 876.25, "completions/mean_terminated_length": 630.0000610351562, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.08073249459266663, "epoch": 0.2838452787258248, "frac_reward_zero_std": 0.0, "grad_norm": 0.01851324923336506, "kl": 0.008776663278695196, "learning_rate": 6.450511945392491e-06, "loss": 0.0383, "num_tokens": 53189898.0, "reward": 0.699999988079071, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.3548576831817627, "sampling/importance_sampling_ratio/mean": 0.4015888571739197, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8761334419250488, "sampling/sampling_logp_difference/mean": 0.010845805518329144, "step": 499, "step_time": 74.17351782112382 }, { "clip_ratio/high_max": 0.010115533194039017, "clip_ratio/high_mean": 0.0026940763636957854, "clip_ratio/low_mean": 0.002587890630820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00528196690720506, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 811.0, "completions/mean_length": 787.6000366210938, "completions/mean_terminated_length": 498.6666564941406, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.1368373967707157, "epoch": 0.2844141069397042, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03468290716409683, "kl": 0.010196072806138545, "learning_rate": 6.4453924914675776e-06, "loss": -0.0468, "num_tokens": 53298002.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.687861919403076, "sampling/importance_sampling_ratio/mean": 0.4204622209072113, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.402822732925415, "sampling/sampling_logp_difference/mean": 0.01489325799047947, "step": 500, "step_time": 73.07513820310123 }, { "clip_ratio/high_max": 0.011920786695554852, "clip_ratio/high_mean": 0.0028448285593185574, "clip_ratio/low_mean": 0.0013671875130967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004212015948724002, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 881.7000122070312, "completions/mean_terminated_length": 506.54547119140625, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.1018792362883687, "epoch": 0.28498293515358364, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.017674658447504044, "kl": 0.016708410053979605, "learning_rate": 6.440273037542662e-06, "loss": -0.019, "num_tokens": 53395238.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 1.8826299905776978, "sampling/importance_sampling_ratio/mean": 0.4472265839576721, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.659567356109619, "sampling/sampling_logp_difference/mean": 0.012179436162114143, "step": 501, "step_time": 70.90453321416862 }, { "clip_ratio/high_max": 0.011148257879540324, "clip_ratio/high_mean": 0.0027370953757781535, "clip_ratio/low_mean": 0.0021972656541038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004934360855259001, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 868.125, "completions/mean_terminated_length": 695.8421020507812, "completions/min_length": 431.0, "completions/min_terminated_length": 431.0, "entropy": 0.10760167799890041, "epoch": 0.285551763367463, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04522642865777016, "kl": 0.010308289201930165, "learning_rate": 6.435153583617748e-06, "loss": -0.1513, "num_tokens": 53495235.0, "reward": 0.7250000238418579, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.6400017738342285, "sampling/importance_sampling_ratio/mean": 0.5525745153427124, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.338296413421631, "sampling/sampling_logp_difference/mean": 0.012326201424002647, "step": 502, "step_time": 71.56262018182315 }, { "clip_ratio/high_max": 0.013498520420398563, "clip_ratio/high_mean": 0.004276147243217565, "clip_ratio/low_mean": 0.0015608062712999526, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005836953496327624, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 839.9750366210938, "completions/mean_terminated_length": 615.0555419921875, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.09698767866939306, "epoch": 0.28612059158134245, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02171204425394535, "kl": 0.013541916152462363, "learning_rate": 6.430034129692833e-06, "loss": -0.0102, "num_tokens": 53610202.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.9482580423355103, "sampling/importance_sampling_ratio/mean": 0.344092458486557, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3896102905273438, "sampling/sampling_logp_difference/mean": 0.0128178047016263, "step": 503, "step_time": 75.2526416128967 }, { "clip_ratio/high_max": 0.013295161654241383, "clip_ratio/high_mean": 0.0039016093651298434, "clip_ratio/low_mean": 0.0018066405900754035, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005708250042516738, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 828.2000122070312, "completions/mean_terminated_length": 611.7894897460938, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.11421652138233185, "epoch": 0.28668941979522183, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.05748031660914421, "kl": 0.015286200563423336, "learning_rate": 6.4249146757679185e-06, "loss": -0.1291, "num_tokens": 53726666.0, "reward": 0.925000011920929, "reward_std": 0.9710555076599121, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.5348565578460693, "sampling/importance_sampling_ratio/mean": 0.29242265224456787, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.703434467315674, "sampling/sampling_logp_difference/mean": 0.014896228909492493, "step": 504, "step_time": 75.75594166200608 }, { "clip_ratio/high_max": 0.013531257340218872, "clip_ratio/high_mean": 0.003223067629733123, "clip_ratio/low_mean": 0.0011962890512222657, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004419356700964272, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 831.0, "completions/mean_length": 903.7000122070312, "completions/mean_terminated_length": 586.5454711914062, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "entropy": 0.0963464230298996, "epoch": 0.28725824800910127, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.030238473787903786, "kl": 0.010429832735098898, "learning_rate": 6.419795221843003e-06, "loss": -0.0255, "num_tokens": 53839054.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.4705770015716553, "sampling/importance_sampling_ratio/mean": 0.48938998579978943, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5624241828918457, "sampling/sampling_logp_difference/mean": 0.01166556216776371, "step": 505, "step_time": 75.36530329775997 }, { "clip_ratio/high_max": 0.013644536375068128, "clip_ratio/high_mean": 0.004999319644412026, "clip_ratio/low_mean": 0.003027343802386895, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00802666344679892, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 784.125, "completions/mean_terminated_length": 606.8261108398438, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.1736464984714985, "epoch": 0.28782707622298065, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02664422057569027, "kl": 0.01987159438431263, "learning_rate": 6.4146757679180885e-06, "loss": -0.0002, "num_tokens": 53948035.0, "reward": 0.875, "reward_std": 0.8529735803604126, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.280498504638672, "sampling/importance_sampling_ratio/mean": 0.32018882036209106, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.555722951889038, "sampling/sampling_logp_difference/mean": 0.018951084464788437, "step": 506, "step_time": 73.62213214160874 }, { "clip_ratio/high_max": 0.011600161204114556, "clip_ratio/high_mean": 0.004259143694071099, "clip_ratio/low_mean": 0.002469711056619417, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006728854903485626, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 911.0, "completions/mean_length": 888.5250244140625, "completions/mean_terminated_length": 607.1538696289062, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.1626614760607481, "epoch": 0.2883959044368601, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04184838756918907, "kl": 0.013896092772483826, "learning_rate": 6.409556313993174e-06, "loss": -0.1083, "num_tokens": 54063456.0, "reward": 0.574999988079071, "reward_std": 0.9026315212249756, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.932098627090454, "sampling/importance_sampling_ratio/mean": 0.3834167420864105, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4278640747070312, "sampling/sampling_logp_difference/mean": 0.016558075323700905, "step": 507, "step_time": 75.41465410240926 }, { "clip_ratio/high_max": 0.01510946050984785, "clip_ratio/high_mean": 0.004761153919389471, "clip_ratio/low_mean": 0.001611328130820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006372481875587255, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 823.0, "completions/mean_length": 818.5750122070312, "completions/mean_terminated_length": 510.4375, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.08966427855193615, "epoch": 0.28896473265073946, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013767250813543797, "kl": 0.010703523003030568, "learning_rate": 6.40443686006826e-06, "loss": 0.0349, "num_tokens": 54161863.0, "reward": 0.6000000238418579, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.6231746673583984, "sampling/importance_sampling_ratio/mean": 0.4122953414916992, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.458082675933838, "sampling/sampling_logp_difference/mean": 0.010985933244228363, "step": 508, "step_time": 70.98060961416923 }, { "clip_ratio/high_max": 0.008180488715879619, "clip_ratio/high_mean": 0.0016360977460863069, "clip_ratio/low_mean": 0.0006347656308207661, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002270863376907073, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 890.0, "completions/mean_length": 749.6749877929688, "completions/mean_terminated_length": 525.227294921875, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.09442015830427408, "epoch": 0.2895335608646189, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.009252695366740227, "kl": 0.011116779991425574, "learning_rate": 6.399317406143345e-06, "loss": -0.0368, "num_tokens": 54259402.0, "reward": 1.100000023841858, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.9164190292358398, "sampling/importance_sampling_ratio/mean": 0.49863630533218384, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.325003147125244, "sampling/sampling_logp_difference/mean": 0.013797265477478504, "step": 509, "step_time": 68.73489846661687 }, { "clip_ratio/high_max": 0.01650180364958942, "clip_ratio/high_mean": 0.00432705256389454, "clip_ratio/low_mean": 0.001342773455689894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005669826117809862, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 767.1749877929688, "completions/mean_terminated_length": 595.9583740234375, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.10108037292957306, "epoch": 0.2901023890784983, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03131411224603653, "kl": 0.012855870765633881, "learning_rate": 6.39419795221843e-06, "loss": 0.0487, "num_tokens": 54365089.0, "reward": 1.2000000476837158, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.4399352073669434, "sampling/importance_sampling_ratio/mean": 0.4517647325992584, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 8.999412536621094, "sampling/sampling_logp_difference/mean": 0.013124984689056873, "step": 510, "step_time": 74.40602345066145 }, { "clip_ratio/high_max": 0.005418220884166658, "clip_ratio/high_mean": 0.0010836441942956299, "clip_ratio/low_mean": 0.0008544922038709046, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019381363708816934, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 893.0, "completions/mean_length": 985.6000366210938, "completions/mean_terminated_length": 640.0, "completions/min_length": 414.0, "completions/min_terminated_length": 414.0, "entropy": 0.08744153147563338, "epoch": 0.2906712172923777, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.015730777755379677, "kl": 0.009244328655768186, "learning_rate": 6.389078498293516e-06, "loss": -0.0285, "num_tokens": 54474345.0, "reward": 0.20000000298023224, "reward_std": 0.6076436042785645, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.954042434692383, "sampling/importance_sampling_ratio/mean": 0.48284703493118286, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.255894184112549, "sampling/sampling_logp_difference/mean": 0.01075300294905901, "step": 511, "step_time": 75.16785215609707 }, { "clip_ratio/high_max": 0.014597221044823527, "clip_ratio/high_mean": 0.0038304617046378553, "clip_ratio/low_mean": 0.0009521484462311491, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0047826100562815554, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 779.0, "completions/mean_length": 861.25, "completions/mean_terminated_length": 432.18182373046875, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.10378505010157824, "epoch": 0.2912400455062571, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01673130691051483, "kl": 0.013728619902394712, "learning_rate": 6.383959044368601e-06, "loss": 0.0303, "num_tokens": 54582027.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.3892649412155151, "sampling/importance_sampling_ratio/mean": 0.3152719736099243, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4980549812316895, "sampling/sampling_logp_difference/mean": 0.011836494319140911, "step": 512, "step_time": 73.06579934363253 }, { "clip_ratio/high_max": 0.009478690335527062, "clip_ratio/high_mean": 0.003170663934724871, "clip_ratio/low_mean": 0.002326324407476932, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005496988364029676, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 846.0499877929688, "completions/mean_terminated_length": 605.2941284179688, "completions/min_length": 404.0, "completions/min_terminated_length": 404.0, "entropy": 0.077133784070611, "epoch": 0.29180887372013653, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02219368703663349, "kl": 0.010410374030470848, "learning_rate": 6.378839590443686e-06, "loss": -0.0232, "num_tokens": 54671949.0, "reward": 0.675000011920929, "reward_std": 0.8589648008346558, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.0149590969085693, "sampling/importance_sampling_ratio/mean": 0.37012726068496704, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4328858852386475, "sampling/sampling_logp_difference/mean": 0.010900384746491909, "step": 513, "step_time": 67.81086042313837 }, { "clip_ratio/high_max": 0.011161484580952674, "clip_ratio/high_mean": 0.003866527898935601, "clip_ratio/low_mean": 0.0013919360026193317, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005258463905192912, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 808.2000122070312, "completions/mean_terminated_length": 664.3333740234375, "completions/min_length": 393.0, "completions/min_terminated_length": 393.0, "entropy": 0.09540591202676296, "epoch": 0.2923777019340159, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019989298656582832, "kl": 0.01729032036382705, "learning_rate": 6.373720136518771e-06, "loss": -0.0007, "num_tokens": 54775413.0, "reward": 1.024999976158142, "reward_std": 0.9195176959037781, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.8905836343765259, "sampling/importance_sampling_ratio/mean": 0.2753133773803711, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2913732528686523, "sampling/sampling_logp_difference/mean": 0.01290081162005663, "step": 514, "step_time": 72.25501788430847 }, { "clip_ratio/high_max": 0.01603551779408008, "clip_ratio/high_mean": 0.0051429760351311415, "clip_ratio/low_mean": 0.002628856571391225, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007771832519210875, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 848.9750366210938, "completions/mean_terminated_length": 586.4375, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.11685102619230747, "epoch": 0.29294653014789535, "frac_reward_zero_std": 0.0, "grad_norm": 0.03387109935283661, "kl": 0.012572433915920556, "learning_rate": 6.368600682593857e-06, "loss": -0.1511, "num_tokens": 54880828.0, "reward": 0.625, "reward_std": 0.8678739070892334, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.8221561908721924, "sampling/importance_sampling_ratio/mean": 0.4863048493862152, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4432692527770996, "sampling/sampling_logp_difference/mean": 0.013595647178590298, "step": 515, "step_time": 74.09633525530808 }, { "clip_ratio/high_max": 0.012344909540843219, "clip_ratio/high_mean": 0.003281851444626227, "clip_ratio/low_mean": 0.0014906932865415001, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00477254469296895, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 886.0499877929688, "completions/mean_terminated_length": 599.5385131835938, "completions/min_length": 442.0, "completions/min_terminated_length": 442.0, "entropy": 0.09809533599764109, "epoch": 0.2935153583617747, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016596559435129166, "kl": 0.010435436794068664, "learning_rate": 6.363481228668943e-06, "loss": -0.0342, "num_tokens": 54977710.0, "reward": 0.5, "reward_std": 0.7844645380973816, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.5281291007995605, "sampling/importance_sampling_ratio/mean": 0.4994603097438812, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3554394245147705, "sampling/sampling_logp_difference/mean": 0.01215110532939434, "step": 516, "step_time": 71.12678157701157 }, { "clip_ratio/high_max": 0.012661155662499368, "clip_ratio/high_mean": 0.0036173333937767893, "clip_ratio/low_mean": 0.0003662109338620212, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003983544302172959, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 858.0, "completions/mean_length": 800.375, "completions/mean_terminated_length": 598.047607421875, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.13349108304828405, "epoch": 0.29408418657565416, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.004852014593780041, "kl": 0.01387386058922857, "learning_rate": 6.358361774744027e-06, "loss": -0.0127, "num_tokens": 55082957.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.9032915830612183, "sampling/importance_sampling_ratio/mean": 0.3034234941005707, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5556721687316895, "sampling/sampling_logp_difference/mean": 0.015389248728752136, "step": 517, "step_time": 73.00591265456751 }, { "clip_ratio/high_max": 0.010973962576827034, "clip_ratio/high_mean": 0.002879663068597438, "clip_ratio/low_mean": 0.002026367212238256, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00490603013895452, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 814.4249877929688, "completions/mean_terminated_length": 604.8500366210938, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.08428443316370249, "epoch": 0.29465301478953354, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027309658005833626, "kl": 0.010658823419362307, "learning_rate": 6.353242320819113e-06, "loss": 0.0327, "num_tokens": 55177438.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.8196669816970825, "sampling/importance_sampling_ratio/mean": 0.49525466561317444, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.94724440574646, "sampling/sampling_logp_difference/mean": 0.011907117441296577, "step": 518, "step_time": 73.23956627305597 }, { "clip_ratio/high_max": 0.006002343638101593, "clip_ratio/high_mean": 0.002049072900263127, "clip_ratio/low_mean": 0.0011718750065483619, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032209479104494676, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 705.1749877929688, "completions/mean_terminated_length": 492.625, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.07375215506181121, "epoch": 0.295221843003413, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030661864206194878, "kl": 0.018765149754472077, "learning_rate": 6.348122866894198e-06, "loss": 0.0268, "num_tokens": 55270605.0, "reward": 1.100000023841858, "reward_std": 0.9554163813591003, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.198026657104492, "sampling/importance_sampling_ratio/mean": 0.4349239468574524, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4529223442077637, "sampling/sampling_logp_difference/mean": 0.010918335057795048, "step": 519, "step_time": 68.28082386008464 }, { "clip_ratio/high_max": 0.008801597403362393, "clip_ratio/high_mean": 0.002702161800698377, "clip_ratio/low_mean": 0.0003662109411379788, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030683727527502924, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 696.6000366210938, "completions/mean_terminated_length": 538.9629516601562, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "entropy": 0.0746858399361372, "epoch": 0.29579067121729236, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009081771597266197, "kl": 0.015550231211818755, "learning_rate": 6.343003412969284e-06, "loss": -0.0229, "num_tokens": 55376725.0, "reward": 1.350000023841858, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.276383399963379, "sampling/importance_sampling_ratio/mean": 0.3010183274745941, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.661700963973999, "sampling/sampling_logp_difference/mean": 0.012121270410716534, "step": 520, "step_time": 72.4763190059457 }, { "clip_ratio/high_max": 0.010570855345577002, "clip_ratio/high_mean": 0.002692626731004566, "clip_ratio/low_mean": 0.0005126953219587449, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032053220293164486, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 830.2750244140625, "completions/mean_terminated_length": 593.5, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.12210284266620874, "epoch": 0.2963594994311718, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.010602058842778206, "kl": 0.015265963855199516, "learning_rate": 6.337883959044368e-06, "loss": -0.0146, "num_tokens": 55487568.0, "reward": 0.9000000357627869, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.070711374282837, "sampling/importance_sampling_ratio/mean": 0.3255113959312439, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.920973777770996, "sampling/sampling_logp_difference/mean": 0.015021293424069881, "step": 521, "step_time": 74.30361713585444 }, { "clip_ratio/high_max": 0.010751314082881436, "clip_ratio/high_mean": 0.0021502628151210956, "clip_ratio/low_mean": 0.0006835937583673513, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002833856546203606, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 744.7999877929688, "completions/mean_terminated_length": 538.434814453125, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.10450456570833921, "epoch": 0.29692832764505117, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.024283720180392265, "kl": 0.014891850063577294, "learning_rate": 6.332764505119454e-06, "loss": -0.0242, "num_tokens": 55584752.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.4270392656326294, "sampling/importance_sampling_ratio/mean": 0.33261746168136597, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9640558958053589, "sampling/sampling_logp_difference/mean": 0.013928129337728024, "step": 522, "step_time": 72.66353128384799 }, { "clip_ratio/high_max": 0.01328389230184257, "clip_ratio/high_mean": 0.003914663728210144, "clip_ratio/low_mean": 0.0016357421827706276, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005550406000111252, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 794.0499877929688, "completions/mean_terminated_length": 539.8947143554688, "completions/min_length": 414.0, "completions/min_terminated_length": 414.0, "entropy": 0.13145691249519587, "epoch": 0.2974971558589306, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01166380476206541, "kl": 0.016486053820699453, "learning_rate": 6.32764505119454e-06, "loss": 0.0765, "num_tokens": 55695266.0, "reward": 0.7750000357627869, "reward_std": 0.8911963105201721, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.488858699798584, "sampling/importance_sampling_ratio/mean": 0.3405011296272278, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.103454113006592, "sampling/sampling_logp_difference/mean": 0.01554765272885561, "step": 523, "step_time": 74.11123668495566 }, { "clip_ratio/high_max": 0.00830287579447031, "clip_ratio/high_mean": 0.0028174657418276183, "clip_ratio/low_mean": 0.0015380859258584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004355551674962044, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 832.0, "completions/mean_length": 804.2750244140625, "completions/mean_terminated_length": 561.4210815429688, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.13005420845001936, "epoch": 0.29806598407281, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.026143738999962807, "kl": 0.01582295133266598, "learning_rate": 6.3225255972696254e-06, "loss": 0.0881, "num_tokens": 55796893.0, "reward": 0.8500000238418579, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.7912887334823608, "sampling/importance_sampling_ratio/mean": 0.38636404275894165, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0684890747070312, "sampling/sampling_logp_difference/mean": 0.015479795634746552, "step": 524, "step_time": 74.28008404374123 }, { "clip_ratio/high_max": 0.013039839512202889, "clip_ratio/high_mean": 0.004211695530102588, "clip_ratio/low_mean": 0.0016357422155124368, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0058474378602113575, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 930.0, "completions/mean_length": 847.7750244140625, "completions/mean_terminated_length": 520.5, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.06982433423399925, "epoch": 0.2986348122866894, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0412868857383728, "kl": 0.017348855617456138, "learning_rate": 6.31740614334471e-06, "loss": -0.1445, "num_tokens": 55895100.0, "reward": 0.675000011920929, "reward_std": 0.9442809820175171, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.5236058235168457, "sampling/importance_sampling_ratio/mean": 0.3658435642719269, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.127025604248047, "sampling/sampling_logp_difference/mean": 0.010526560246944427, "step": 525, "step_time": 70.41807729448192 }, { "clip_ratio/high_max": 0.01692678628023714, "clip_ratio/high_mean": 0.006844917428679764, "clip_ratio/low_mean": 0.002001953158469405, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008846870623528957, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 752.9000244140625, "completions/mean_terminated_length": 552.521728515625, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.11298155598342419, "epoch": 0.2992036405005688, "frac_reward_zero_std": 0.0, "grad_norm": 0.05656372010707855, "kl": 0.017115769907832146, "learning_rate": 6.3122866894197955e-06, "loss": -0.0513, "num_tokens": 56007920.0, "reward": 0.875, "reward_std": 0.8529736399650574, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.6105153560638428, "sampling/importance_sampling_ratio/mean": 0.44091159105300903, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.721420764923096, "sampling/sampling_logp_difference/mean": 0.014868858270347118, "step": 526, "step_time": 76.44352391641587 }, { "clip_ratio/high_max": 0.009414300438947976, "clip_ratio/high_mean": 0.002529432938899845, "clip_ratio/low_mean": 0.002852843375876546, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00538227628567256, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 894.8250122070312, "completions/mean_terminated_length": 654.9285888671875, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.07080374797806144, "epoch": 0.29977246871444824, "frac_reward_zero_std": 0.0, "grad_norm": 0.023366006091237068, "kl": 0.023306419141590595, "learning_rate": 6.307167235494881e-06, "loss": 0.011, "num_tokens": 56115025.0, "reward": 0.6000000238418579, "reward_std": 0.8711913228034973, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.911031484603882, "sampling/importance_sampling_ratio/mean": 0.4162062108516693, "sampling/importance_sampling_ratio/min": 0.00018377961532678455, "sampling/sampling_logp_difference/max": 3.272824764251709, "sampling/sampling_logp_difference/mean": 0.011117632500827312, "step": 527, "step_time": 74.4409567117691 }, { "clip_ratio/high_max": 0.006554915045853704, "clip_ratio/high_mean": 0.0015957210707711056, "clip_ratio/low_mean": 0.0012939453299622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028896663716295734, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 897.9249877929688, "completions/mean_terminated_length": 687.800048828125, "completions/min_length": 439.0, "completions/min_terminated_length": 439.0, "entropy": 0.14806235395371914, "epoch": 0.3003412969283277, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.014113208279013634, "kl": 0.011351322173140943, "learning_rate": 6.302047781569966e-06, "loss": 0.0204, "num_tokens": 56216326.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.131998300552368, "sampling/importance_sampling_ratio/mean": 0.4127840995788574, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7746577262878418, "sampling/sampling_logp_difference/mean": 0.016184860840439796, "step": 528, "step_time": 72.38071292988025 }, { "clip_ratio/high_max": 0.012989008057047613, "clip_ratio/high_mean": 0.003865338720061118, "clip_ratio/low_mean": 0.0011994688047707314, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005064807541202754, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 796.875, "completions/mean_terminated_length": 611.0454711914062, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.08750852942466736, "epoch": 0.30091012514220705, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04065107926726341, "kl": 0.01585819397587329, "learning_rate": 6.296928327645051e-06, "loss": 0.0425, "num_tokens": 56323241.0, "reward": 1.0750000476837158, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.9947447776794434, "sampling/importance_sampling_ratio/mean": 0.6560443043708801, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.147289276123047, "sampling/sampling_logp_difference/mean": 0.012247788719832897, "step": 529, "step_time": 74.58713289303705 }, { "clip_ratio/high_max": 0.014562942000338808, "clip_ratio/high_mean": 0.0036807820652029477, "clip_ratio/low_mean": 0.002286207520228345, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005966989672742784, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 827.0, "completions/mean_length": 835.625, "completions/mean_terminated_length": 521.6666870117188, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.0898561766371131, "epoch": 0.3014789533560865, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04465872794389725, "kl": 0.010384385648649186, "learning_rate": 6.291808873720136e-06, "loss": -0.0805, "num_tokens": 56428946.0, "reward": 0.6500000357627869, "reward_std": 0.9212589263916016, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.7135112285614014, "sampling/importance_sampling_ratio/mean": 0.48245635628700256, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7357052564620972, "sampling/sampling_logp_difference/mean": 0.012066434137523174, "step": 530, "step_time": 73.03698955662549 }, { "clip_ratio/high_max": 0.010825719131389633, "clip_ratio/high_mean": 0.003959435241995379, "clip_ratio/low_mean": 0.0024169921853172127, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006376427336363122, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 780.4750366210938, "completions/mean_terminated_length": 581.227294921875, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.17057637125253677, "epoch": 0.30204778156996587, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01687500812113285, "kl": 0.01584102597553283, "learning_rate": 6.286689419795223e-06, "loss": 0.0124, "num_tokens": 56531253.0, "reward": 1.0, "reward_std": 0.9607689380645752, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.7865904569625854, "sampling/importance_sampling_ratio/mean": 0.31435275077819824, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0252816677093506, "sampling/sampling_logp_difference/mean": 0.019057542085647583, "step": 531, "step_time": 69.98626713082194 }, { "clip_ratio/high_max": 0.003925187571439892, "clip_ratio/high_mean": 0.0007850375259295106, "clip_ratio/low_mean": 0.0012939453372382559, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002078982812236063, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 776.5, "completions/mean_terminated_length": 529.0, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.12328009679913521, "epoch": 0.3026166097838453, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009003587998449802, "kl": 0.014746764791198075, "learning_rate": 6.281569965870307e-06, "loss": -0.0217, "num_tokens": 56633641.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.0540186166763306, "sampling/importance_sampling_ratio/mean": 0.2654196619987488, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.174382209777832, "sampling/sampling_logp_difference/mean": 0.014505714178085327, "step": 532, "step_time": 71.0698871195782 }, { "clip_ratio/high_max": 0.007216520432848483, "clip_ratio/high_mean": 0.0017005429399432614, "clip_ratio/low_mean": 0.00031738281541038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0020179257589916233, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 803.7999877929688, "completions/mean_terminated_length": 505.8823547363281, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.06927799386903644, "epoch": 0.3031854379977247, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.018781693652272224, "kl": 0.010385287052486092, "learning_rate": 6.276450511945393e-06, "loss": -0.0351, "num_tokens": 56739585.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.6309456825256348, "sampling/importance_sampling_ratio/mean": 0.48970532417297363, "sampling/importance_sampling_ratio/min": 0.019944334402680397, "sampling/sampling_logp_difference/max": 2.4787590503692627, "sampling/sampling_logp_difference/mean": 0.010519303381443024, "step": 533, "step_time": 71.50183576229028 }, { "clip_ratio/high_max": 0.014175419579260051, "clip_ratio/high_mean": 0.005945481272647157, "clip_ratio/low_mean": 0.0010366586466261651, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.006982139952015132, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 828.1749877929688, "completions/mean_terminated_length": 697.625, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.08710893429815769, "epoch": 0.3037542662116041, "frac_reward_zero_std": 0.0, "grad_norm": 0.012635542079806328, "kl": 0.014857314759865403, "learning_rate": 6.271331058020478e-06, "loss": -0.0668, "num_tokens": 56838992.0, "reward": 1.1750000715255737, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.918613314628601, "sampling/importance_sampling_ratio/mean": 0.28535693883895874, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9724974632263184, "sampling/sampling_logp_difference/mean": 0.012319790199398994, "step": 534, "step_time": 73.32119562686421 }, { "clip_ratio/high_max": 0.01764893438667059, "clip_ratio/high_mean": 0.007033797388430685, "clip_ratio/low_mean": 0.0032861196086741984, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010319916880689561, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 699.4750366210938, "completions/mean_terminated_length": 543.2222290039062, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.13751804642379284, "epoch": 0.3043230944254835, "frac_reward_zero_std": 0.0, "grad_norm": 0.029872562736272812, "kl": 0.02192282967735082, "learning_rate": 6.2662116040955635e-06, "loss": -0.0754, "num_tokens": 56939283.0, "reward": 1.25, "reward_std": 0.9268086552619934, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.155461311340332, "sampling/importance_sampling_ratio/mean": 0.377654105424881, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.325002908706665, "sampling/sampling_logp_difference/mean": 0.018014924600720406, "step": 535, "step_time": 69.5477747595869 }, { "clip_ratio/high_max": 0.015962437610141933, "clip_ratio/high_mean": 0.00501517512020655, "clip_ratio/low_mean": 0.0020751953361468622, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0070903702871873975, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 815.9249877929688, "completions/mean_terminated_length": 645.6818237304688, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.12582488916814327, "epoch": 0.30489192263936293, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.046759817749261856, "kl": 0.01615438691806048, "learning_rate": 6.261092150170648e-06, "loss": 0.0801, "num_tokens": 57047624.0, "reward": 1.024999976158142, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.68550968170166, "sampling/importance_sampling_ratio/mean": 0.34225302934646606, "sampling/importance_sampling_ratio/min": 0.002933061681687832, "sampling/sampling_logp_difference/max": 2.753868579864502, "sampling/sampling_logp_difference/mean": 0.015777671709656715, "step": 536, "step_time": 74.32785219699144 }, { "clip_ratio/high_max": 0.014770307461731136, "clip_ratio/high_mean": 0.0045694355794694275, "clip_ratio/low_mean": 0.0032133958011399955, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007782831438817084, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 906.0, "completions/mean_length": 864.25, "completions/mean_terminated_length": 624.625, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.12583993282169104, "epoch": 0.3054607508532423, "frac_reward_zero_std": 0.0, "grad_norm": 0.02925930730998516, "kl": 0.01336550759151578, "learning_rate": 6.2559726962457335e-06, "loss": -0.0036, "num_tokens": 57154450.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.7116453647613525, "sampling/importance_sampling_ratio/mean": 0.42863303422927856, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9860265254974365, "sampling/sampling_logp_difference/mean": 0.015559809282422066, "step": 537, "step_time": 71.93298248574138 }, { "clip_ratio/high_max": 0.004903466207906604, "clip_ratio/high_mean": 0.0011249933450017124, "clip_ratio/low_mean": 0.002612304684589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037372980441432446, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 866.6749877929688, "completions/mean_terminated_length": 499.5833435058594, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.09701340738683939, "epoch": 0.30602957906712175, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02308649569749832, "kl": 0.011599024408496916, "learning_rate": 6.250853242320819e-06, "loss": 0.0317, "num_tokens": 57265845.0, "reward": 0.574999988079071, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 1.8458410501480103, "sampling/importance_sampling_ratio/mean": 0.3560826778411865, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5775678157806396, "sampling/sampling_logp_difference/mean": 0.012682775035500526, "step": 538, "step_time": 74.10566747025587 }, { "clip_ratio/high_max": 0.0107397208630573, "clip_ratio/high_mean": 0.003941745504562277, "clip_ratio/low_mean": 0.001147460934589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005089206475531682, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 614.0, "completions/mean_length": 799.3250122070312, "completions/mean_terminated_length": 382.0714416503906, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.0705197206698358, "epoch": 0.30659840728100113, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026547038927674294, "kl": 0.019506138865835965, "learning_rate": 6.245733788395905e-06, "loss": 0.0858, "num_tokens": 57356954.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.9123098850250244, "sampling/importance_sampling_ratio/mean": 0.5170853734016418, "sampling/importance_sampling_ratio/min": 0.0013814811827614903, "sampling/sampling_logp_difference/max": 4.926954746246338, "sampling/sampling_logp_difference/mean": 0.0096755251288414, "step": 539, "step_time": 68.46589673939161 }, { "clip_ratio/high_max": 0.014773062779568136, "clip_ratio/high_mean": 0.004986790791008389, "clip_ratio/low_mean": 0.0018866292957682163, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006873419973999262, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 971.0, "completions/mean_length": 761.75, "completions/mean_terminated_length": 567.9130249023438, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.10065555386245251, "epoch": 0.30716723549488056, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.07981116324663162, "kl": 0.011175761930644512, "learning_rate": 6.24061433447099e-06, "loss": -0.0264, "num_tokens": 57445960.0, "reward": 0.9750000238418579, "reward_std": 0.9195176959037781, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.8150362968444824, "sampling/importance_sampling_ratio/mean": 0.5818977952003479, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5078766345977783, "sampling/sampling_logp_difference/mean": 0.013723931275308132, "step": 540, "step_time": 70.5297830409836 }, { "clip_ratio/high_max": 0.015783907612785697, "clip_ratio/high_mean": 0.005116288157296367, "clip_ratio/low_mean": 0.00283203118306119, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007948319369461387, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 816.0, "completions/mean_length": 850.7250366210938, "completions/mean_terminated_length": 561.933349609375, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.10819408670067787, "epoch": 0.30773606370875994, "frac_reward_zero_std": 0.0, "grad_norm": 0.09058431535959244, "kl": 0.01687995542306453, "learning_rate": 6.235494880546075e-06, "loss": 0.2354, "num_tokens": 57552181.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.6633031368255615, "sampling/importance_sampling_ratio/mean": 0.6206464171409607, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.865973949432373, "sampling/sampling_logp_difference/mean": 0.013432571664452553, "step": 541, "step_time": 71.94583112793043 }, { "clip_ratio/high_max": 0.012671206146478653, "clip_ratio/high_mean": 0.00383581334608607, "clip_ratio/low_mean": 0.0029469708388205618, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0067827843595296144, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 851.1500244140625, "completions/mean_terminated_length": 678.2999877929688, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.07293360400944948, "epoch": 0.3083048919226394, "frac_reward_zero_std": 0.0, "grad_norm": 0.026398690417408943, "kl": 0.010692032752558589, "learning_rate": 6.230375426621161e-06, "loss": 0.02, "num_tokens": 57655171.0, "reward": 0.675000011920929, "reward_std": 0.7641821503639221, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.8447526693344116, "sampling/importance_sampling_ratio/mean": 0.3133068084716797, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1922101974487305, "sampling/sampling_logp_difference/mean": 0.010787716135382652, "step": 542, "step_time": 73.58665738999844 }, { "clip_ratio/high_max": 0.01597918150946498, "clip_ratio/high_mean": 0.0061371861374937, "clip_ratio/low_mean": 0.0015380859622382559, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007675272005144507, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 718.75, "completions/mean_terminated_length": 554.3846435546875, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.12748540099710226, "epoch": 0.30887372013651876, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029113182798027992, "kl": 0.016038190224207938, "learning_rate": 6.225255972696246e-06, "loss": -0.0489, "num_tokens": 57763025.0, "reward": 1.25, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 1.9957822561264038, "sampling/importance_sampling_ratio/mean": 0.3710653483867645, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.08522629737854, "sampling/sampling_logp_difference/mean": 0.01619902066886425, "step": 543, "step_time": 72.84291209373623 }, { "clip_ratio/high_max": 0.010921438806690276, "clip_ratio/high_mean": 0.003119582441286184, "clip_ratio/low_mean": 0.0018554687267169356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004975051153451204, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 952.9750366210938, "completions/mean_terminated_length": 739.9000244140625, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.11083953361958265, "epoch": 0.3094425483503982, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.032558634877204895, "kl": 0.010876703774556518, "learning_rate": 6.220136518771331e-06, "loss": -0.0593, "num_tokens": 57869296.0, "reward": 0.42500001192092896, "reward_std": 0.780778706073761, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.948476552963257, "sampling/importance_sampling_ratio/mean": 0.4317046105861664, "sampling/importance_sampling_ratio/min": 0.0019745936151593924, "sampling/sampling_logp_difference/max": 1.9563231468200684, "sampling/sampling_logp_difference/mean": 0.012390680611133575, "step": 544, "step_time": 74.72752966289409 }, { "clip_ratio/high_max": 0.0030573419644497335, "clip_ratio/high_mean": 0.0006114684001659043, "clip_ratio/low_mean": 0.0016845703503349796, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022960387432249263, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 920.0, "completions/mean_length": 886.6500244140625, "completions/mean_terminated_length": 524.5454711914062, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.0914460476487875, "epoch": 0.3100113765642776, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009374996647238731, "kl": 0.011750871723052114, "learning_rate": 6.215017064846416e-06, "loss": -0.0104, "num_tokens": 57979738.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.491821527481079, "sampling/importance_sampling_ratio/mean": 0.5372188687324524, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9558486938476562, "sampling/sampling_logp_difference/mean": 0.012089909054338932, "step": 545, "step_time": 72.52338102646172 }, { "clip_ratio/high_max": 0.012828110076952726, "clip_ratio/high_mean": 0.003915045715984888, "clip_ratio/low_mean": 0.0013885840962757356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005303629790432751, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 802.0, "completions/mean_length": 785.8500366210938, "completions/mean_terminated_length": 591.0, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.10945022013038397, "epoch": 0.310580204778157, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011247682385146618, "kl": 0.016489326837472618, "learning_rate": 6.209897610921502e-06, "loss": 0.017, "num_tokens": 58090460.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.7953088283538818, "sampling/importance_sampling_ratio/mean": 0.2966746687889099, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.142650842666626, "sampling/sampling_logp_difference/mean": 0.01450644712895155, "step": 546, "step_time": 72.8345014473889 }, { "clip_ratio/high_max": 0.002954068360850215, "clip_ratio/high_mean": 0.0005908136699872557, "clip_ratio/low_mean": 0.0014648437463620212, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002055657430901192, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 815.0250244140625, "completions/mean_terminated_length": 584.0526123046875, "completions/min_length": 424.0, "completions/min_terminated_length": 424.0, "entropy": 0.11711735744029284, "epoch": 0.3111490329920364, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.018547510728240013, "kl": 0.015317957964725792, "learning_rate": 6.204778156996588e-06, "loss": -0.0205, "num_tokens": 58196957.0, "reward": 0.9000000357627869, "reward_std": 0.9818872213363647, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.4047484397888184, "sampling/importance_sampling_ratio/mean": 0.3405407965183258, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.073002815246582, "sampling/sampling_logp_difference/mean": 0.014931783080101013, "step": 547, "step_time": 72.18321195128374 }, { "clip_ratio/high_max": 0.011773496400564909, "clip_ratio/high_mean": 0.0034061117912642658, "clip_ratio/low_mean": 0.0013360642697080038, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004742176010040566, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 779.25, "completions/mean_terminated_length": 579.0, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.09060750622302294, "epoch": 0.3117178612059158, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.020404381677508354, "kl": 0.01674078532960266, "learning_rate": 6.1996587030716725e-06, "loss": 0.0106, "num_tokens": 58303671.0, "reward": 1.0500000715255737, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.721908688545227, "sampling/importance_sampling_ratio/mean": 0.3772309422492981, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1184329986572266, "sampling/sampling_logp_difference/mean": 0.013101157732307911, "step": 548, "step_time": 70.5905168117024 }, { "clip_ratio/high_max": 0.008769016189035028, "clip_ratio/high_mean": 0.002380023041041568, "clip_ratio/low_mean": 0.0033691406424622983, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.005749163625296205, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 853.0, "completions/mean_length": 912.875, "completions/mean_terminated_length": 619.9091186523438, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "entropy": 0.11190229468047619, "epoch": 0.3122866894197952, "frac_reward_zero_std": 0.0, "grad_norm": 0.04791649430990219, "kl": 0.011406783829443157, "learning_rate": 6.194539249146758e-06, "loss": -0.1871, "num_tokens": 58413938.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.6234731674194336, "sampling/importance_sampling_ratio/mean": 0.4649578928947449, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0221152305603027, "sampling/sampling_logp_difference/mean": 0.013106402941048145, "step": 549, "step_time": 75.37479739752598 }, { "clip_ratio/high_max": 0.014137020683847368, "clip_ratio/high_mean": 0.0038833353100926615, "clip_ratio/low_mean": 0.0012471986410673708, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005130533914780244, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 821.9750366210938, "completions/mean_terminated_length": 724.7037353515625, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 0.18776923790574074, "epoch": 0.31285551763367464, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019873103126883507, "kl": 0.01451588305644691, "learning_rate": 6.189419795221843e-06, "loss": 0.0463, "num_tokens": 58515097.0, "reward": 1.2000000476837158, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.4430519342422485, "sampling/importance_sampling_ratio/mean": 0.3348146975040436, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.104757785797119, "sampling/sampling_logp_difference/mean": 0.019144538789987564, "step": 550, "step_time": 75.75052539329045 }, { "clip_ratio/high_max": 0.012430411938112229, "clip_ratio/high_mean": 0.0032457460038131103, "clip_ratio/low_mean": 0.0022460937325377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005491839809110388, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 969.0, "completions/mean_length": 877.1000366210938, "completions/mean_terminated_length": 534.3333740234375, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.0710183666087687, "epoch": 0.313424345847554, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.028060803189873695, "kl": 0.01110072759911418, "learning_rate": 6.184300341296929e-06, "loss": -0.0654, "num_tokens": 58620853.0, "reward": 0.5, "reward_std": 0.8164966106414795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.840487241744995, "sampling/importance_sampling_ratio/mean": 0.48365506529808044, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2320396900177, "sampling/sampling_logp_difference/mean": 0.00960760097950697, "step": 551, "step_time": 72.9869161369279 }, { "clip_ratio/high_max": 0.007589508546516299, "clip_ratio/high_mean": 0.0018195054144598544, "clip_ratio/low_mean": 0.0019281364366179332, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003747641821973957, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 883.625, "completions/mean_terminated_length": 462.5, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.110911063849926, "epoch": 0.31399317406143346, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02039036713540554, "kl": 0.011561451363377273, "learning_rate": 6.179180887372013e-06, "loss": 0.0976, "num_tokens": 58730902.0, "reward": 0.42500001192092896, "reward_std": 0.780778706073761, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.2401905059814453, "sampling/importance_sampling_ratio/mean": 0.32833918929100037, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.516205310821533, "sampling/sampling_logp_difference/mean": 0.013602052815258503, "step": 552, "step_time": 72.56878810026683 }, { "clip_ratio/high_max": 0.01623465633019805, "clip_ratio/high_mean": 0.007068945415085182, "clip_ratio/low_mean": 0.0010986328197759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008167578431311995, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 773.2999877929688, "completions/mean_terminated_length": 606.1666870117188, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.09986744541674852, "epoch": 0.31456200227531284, "frac_reward_zero_std": 0.0, "grad_norm": 0.039080433547496796, "kl": 0.016611250874120742, "learning_rate": 6.174061433447099e-06, "loss": -0.0082, "num_tokens": 58833666.0, "reward": 1.2000000476837158, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.321026563644409, "sampling/importance_sampling_ratio/mean": 0.4076870083808899, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1079258918762207, "sampling/sampling_logp_difference/mean": 0.013088616542518139, "step": 553, "step_time": 74.12969226273708 }, { "clip_ratio/high_max": 0.012697827070951462, "clip_ratio/high_mean": 0.0035812627320410684, "clip_ratio/low_mean": 0.003247070300858468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006828333076555282, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 914.9249877929688, "completions/mean_terminated_length": 688.3846435546875, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.10868404991924763, "epoch": 0.31513083048919227, "frac_reward_zero_std": 0.0, "grad_norm": 0.030565716326236725, "kl": 0.011574366770219058, "learning_rate": 6.168941979522184e-06, "loss": 0.1084, "num_tokens": 58951159.0, "reward": 0.5, "reward_std": 0.7844645380973816, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.5406224727630615, "sampling/importance_sampling_ratio/mean": 0.5855320692062378, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2424914836883545, "sampling/sampling_logp_difference/mean": 0.011859116144478321, "step": 554, "step_time": 77.50797530799173 }, { "clip_ratio/high_max": 0.011903889710083604, "clip_ratio/high_mean": 0.0030210896220523864, "clip_ratio/low_mean": 0.002636718774738256, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005657808447722346, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 901.1749877929688, "completions/mean_terminated_length": 751.0555419921875, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 0.149479566141963, "epoch": 0.31569965870307165, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.016383212059736252, "kl": 0.012757382239215076, "learning_rate": 6.1638225255972705e-06, "loss": -0.0247, "num_tokens": 59065190.0, "reward": 0.699999988079071, "reward_std": 0.8533493876457214, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.6262367963790894, "sampling/importance_sampling_ratio/mean": 0.23501995205879211, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.043757915496826, "sampling/sampling_logp_difference/mean": 0.015464507043361664, "step": 555, "step_time": 75.17996127577499 }, { "clip_ratio/high_max": 0.012607285287231207, "clip_ratio/high_mean": 0.0029569971084129065, "clip_ratio/low_mean": 0.002270507837238256, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005227504996582866, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 983.0499877929688, "completions/mean_terminated_length": 790.0000610351562, "completions/min_length": 574.0, "completions/min_terminated_length": 574.0, "entropy": 0.20470758341252804, "epoch": 0.3162684869169511, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011763009242713451, "kl": 0.013309438596479595, "learning_rate": 6.158703071672355e-06, "loss": 0.0586, "num_tokens": 59183832.0, "reward": 0.2750000059604645, "reward_std": 0.6400120258331299, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 1.525138020515442, "sampling/importance_sampling_ratio/mean": 0.2940387725830078, "sampling/importance_sampling_ratio/min": 0.0005319262272678316, "sampling/sampling_logp_difference/max": 2.118664503097534, "sampling/sampling_logp_difference/mean": 0.018113000318408012, "step": 556, "step_time": 77.59418977680616 }, { "clip_ratio/high_max": 0.013624757062643766, "clip_ratio/high_mean": 0.004743199504446238, "clip_ratio/low_mean": 0.0019287109425931703, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006671910436125472, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 872.2750244140625, "completions/mean_terminated_length": 619.4000244140625, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.10684916004538536, "epoch": 0.31683731513083047, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014923596754670143, "kl": 0.01374949817545712, "learning_rate": 6.1535836177474405e-06, "loss": -0.0363, "num_tokens": 59291795.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.33262038230896, "sampling/importance_sampling_ratio/mean": 0.29459208250045776, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.945139169692993, "sampling/sampling_logp_difference/mean": 0.01250514853745699, "step": 557, "step_time": 72.84432882978581 }, { "clip_ratio/high_max": 0.01430448703467846, "clip_ratio/high_mean": 0.005909063038416207, "clip_ratio/low_mean": 0.0010986328125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007007695618085563, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 609.9750366210938, "completions/mean_terminated_length": 471.9667053222656, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.12752485927194357, "epoch": 0.3174061433447099, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013460003770887852, "kl": 0.019418971030972898, "learning_rate": 6.148464163822526e-06, "loss": -0.0089, "num_tokens": 59389690.0, "reward": 1.4500000476837158, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.699999988079071, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.562353491783142, "sampling/importance_sampling_ratio/mean": 0.3060404062271118, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 13.873071670532227, "sampling/sampling_logp_difference/mean": 0.01857978105545044, "step": 558, "step_time": 70.58353450312279 }, { "clip_ratio/high_max": 0.013523200061172247, "clip_ratio/high_mean": 0.0036670126137323678, "clip_ratio/low_mean": 0.0005615234440483619, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004228536039590836, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 947.0, "completions/mean_length": 696.6500244140625, "completions/mean_terminated_length": 539.0370483398438, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.11808076221495867, "epoch": 0.3179749715585893, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.015675053000450134, "kl": 0.01816413493361324, "learning_rate": 6.143344709897611e-06, "loss": -0.0549, "num_tokens": 59494036.0, "reward": 1.350000023841858, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.5216734409332275, "sampling/importance_sampling_ratio/mean": 0.47759324312210083, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.190728187561035, "sampling/sampling_logp_difference/mean": 0.015579983592033386, "step": 559, "step_time": 73.59785222867504 }, { "clip_ratio/high_max": 0.007280243415152654, "clip_ratio/high_mean": 0.00161075170035474, "clip_ratio/low_mean": 0.0013176618595025502, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002928413523477502, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 909.0, "completions/mean_length": 918.0250244140625, "completions/mean_terminated_length": 600.1000366210938, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.1337396316230297, "epoch": 0.3185437997724687, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03052123822271824, "kl": 0.011056285176891834, "learning_rate": 6.138225255972696e-06, "loss": -0.0871, "num_tokens": 59599101.0, "reward": 0.4749999940395355, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.2857019901275635, "sampling/importance_sampling_ratio/mean": 0.33541303873062134, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2361836433410645, "sampling/sampling_logp_difference/mean": 0.014137794263660908, "step": 560, "step_time": 73.94112012162805 }, { "clip_ratio/high_max": 0.014836889458820224, "clip_ratio/high_mean": 0.0036979674478061497, "clip_ratio/low_mean": 0.0031982421714928932, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0068962095538154244, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 802.0, "completions/mean_length": 882.6000366210938, "completions/mean_terminated_length": 552.6666870117188, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.13453984633088112, "epoch": 0.3191126279863481, "frac_reward_zero_std": 0.0, "grad_norm": 0.031521476805210114, "kl": 0.009952919383067638, "learning_rate": 6.133105802047781e-06, "loss": -0.0136, "num_tokens": 59710557.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.9163825511932373, "sampling/importance_sampling_ratio/mean": 0.4114398956298828, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1190319061279297, "sampling/sampling_logp_difference/mean": 0.01375302392989397, "step": 561, "step_time": 76.87311176094227 }, { "clip_ratio/high_max": 0.01061573124025017, "clip_ratio/high_mean": 0.002479337912518531, "clip_ratio/low_mean": 0.002734375011641532, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005213712953263894, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 877.875, "completions/mean_terminated_length": 658.6875, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "entropy": 0.11270786821842194, "epoch": 0.31968145620022753, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.016912302002310753, "kl": 0.013569476897828281, "learning_rate": 6.127986348122868e-06, "loss": -0.0167, "num_tokens": 59808448.0, "reward": 0.699999988079071, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.793194055557251, "sampling/importance_sampling_ratio/mean": 0.443631112575531, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.148794174194336, "sampling/sampling_logp_difference/mean": 0.013170823454856873, "step": 562, "step_time": 71.42661615018733 }, { "clip_ratio/high_max": 0.008815956534817815, "clip_ratio/high_mean": 0.00262400672363583, "clip_ratio/low_mean": 0.0022705077863065526, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004894514480838552, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 960.0, "completions/mean_length": 789.3250122070312, "completions/mean_terminated_length": 632.875, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.1482588481158018, "epoch": 0.32025028441410697, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016826892271637917, "kl": 0.013697561458684504, "learning_rate": 6.122866894197953e-06, "loss": 0.0929, "num_tokens": 59921437.0, "reward": 0.949999988079071, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.34867787361145, "sampling/importance_sampling_ratio/mean": 0.48609447479248047, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5968587398529053, "sampling/sampling_logp_difference/mean": 0.01704305410385132, "step": 563, "step_time": 80.25079284794629 }, { "clip_ratio/high_max": 0.011384321260266006, "clip_ratio/high_mean": 0.003367994708241895, "clip_ratio/low_mean": 0.002026367163125542, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0053943616803735495, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 942.0, "completions/mean_length": 778.8250122070312, "completions/mean_terminated_length": 557.0, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.10282987169921398, "epoch": 0.32081911262798635, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03996177390217781, "kl": 0.012647777562960982, "learning_rate": 6.117747440273038e-06, "loss": 0.1981, "num_tokens": 60021622.0, "reward": 0.9750000238418579, "reward_std": 0.9736924767494202, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.990097999572754, "sampling/importance_sampling_ratio/mean": 0.5187474489212036, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.441990375518799, "sampling/sampling_logp_difference/mean": 0.01321806013584137, "step": 564, "step_time": 69.73227664129809 }, { "clip_ratio/high_max": 0.012591475388035178, "clip_ratio/high_mean": 0.003442824963713065, "clip_ratio/low_mean": 0.0021484375465661287, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005591262481175363, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 933.9249877929688, "completions/mean_terminated_length": 746.84619140625, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.1393798030912876, "epoch": 0.3213879408418658, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.05819079652428627, "kl": 0.012990130460821092, "learning_rate": 6.112627986348123e-06, "loss": -0.1555, "num_tokens": 60130419.0, "reward": 0.550000011920929, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.8624987602233887, "sampling/importance_sampling_ratio/mean": 0.6129940748214722, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7529422044754028, "sampling/sampling_logp_difference/mean": 0.014434856362640858, "step": 565, "step_time": 73.4125497599598 }, { "clip_ratio/high_max": 0.012384677305817604, "clip_ratio/high_mean": 0.003636343462858349, "clip_ratio/low_mean": 0.002148437488358468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005784781009424478, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 843.4750366210938, "completions/mean_terminated_length": 542.6000366210938, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.11680563446134329, "epoch": 0.32195676905574516, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023853465914726257, "kl": 0.012633744743652642, "learning_rate": 6.107508532423209e-06, "loss": 0.0761, "num_tokens": 60234070.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.9790596961975098, "sampling/importance_sampling_ratio/mean": 0.3591022491455078, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.590501546859741, "sampling/sampling_logp_difference/mean": 0.013183879666030407, "step": 566, "step_time": 73.94788764952682 }, { "clip_ratio/high_max": 0.0037817948032170534, "clip_ratio/high_mean": 0.0007814342097844929, "clip_ratio/low_mean": 0.0015136718739086064, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002295106060046237, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 886.5750122070312, "completions/mean_terminated_length": 601.1538696289062, "completions/min_length": 358.0, "completions/min_terminated_length": 358.0, "entropy": 0.07234371872618794, "epoch": 0.3225255972696246, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01515174936503172, "kl": 0.01035046111792326, "learning_rate": 6.102389078498293e-06, "loss": 0.0956, "num_tokens": 60350373.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.193962574005127, "sampling/importance_sampling_ratio/mean": 0.44709429144859314, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7511961460113525, "sampling/sampling_logp_difference/mean": 0.010750234127044678, "step": 567, "step_time": 76.40389477740973 }, { "clip_ratio/high_max": 0.018305495963431895, "clip_ratio/high_mean": 0.006041034561349079, "clip_ratio/low_mean": 0.0011975466186413541, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007238581019919366, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 875.0, "completions/mean_length": 748.6000366210938, "completions/mean_terminated_length": 473.20001220703125, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.11028026975691319, "epoch": 0.323094425483504, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.005407747812569141, "kl": 0.0145256242249161, "learning_rate": 6.097269624573379e-06, "loss": 0.0016, "num_tokens": 60457349.0, "reward": 0.75, "reward_std": 0.8397191166877747, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.6650470495224, "sampling/importance_sampling_ratio/mean": 0.3291890025138855, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8727532625198364, "sampling/sampling_logp_difference/mean": 0.014241923578083515, "step": 568, "step_time": 74.68092633294873 }, { "clip_ratio/high_max": 0.017123758094385266, "clip_ratio/high_mean": 0.006546952063217759, "clip_ratio/low_mean": 0.002098778961226344, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008645731082651764, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 890.0, "completions/mean_length": 676.7999877929688, "completions/mean_terminated_length": 528.0, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.15044864732772112, "epoch": 0.3236632536973834, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01706640049815178, "kl": 0.017544937203638256, "learning_rate": 6.092150170648464e-06, "loss": 0.0841, "num_tokens": 60554149.0, "reward": 1.350000023841858, "reward_std": 0.9212589263916016, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.9655776023864746, "sampling/importance_sampling_ratio/mean": 0.28315943479537964, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.001345157623291, "sampling/sampling_logp_difference/mean": 0.016901595517992973, "step": 569, "step_time": 71.49347188626416 }, { "clip_ratio/high_max": 0.00668444023176562, "clip_ratio/high_mean": 0.0013368880499911029, "clip_ratio/low_mean": 0.00019531250291038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001532200552901486, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 814.0, "completions/mean_length": 767.7250366210938, "completions/mean_terminated_length": 558.0454711914062, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.0943645741790533, "epoch": 0.3242320819112628, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.017061423510313034, "kl": 0.012257836060598493, "learning_rate": 6.08703071672355e-06, "loss": -0.0034, "num_tokens": 60652290.0, "reward": 1.100000023841858, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.2868518829345703, "sampling/importance_sampling_ratio/mean": 0.6095985174179077, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4596505165100098, "sampling/sampling_logp_difference/mean": 0.012642120011150837, "step": 570, "step_time": 68.53407237981446 }, { "clip_ratio/high_max": 0.008215657901018858, "clip_ratio/high_mean": 0.0020049635641044006, "clip_ratio/low_mean": 0.002490234372089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004495197965297848, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 873.2000122070312, "completions/mean_terminated_length": 647.0, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "entropy": 0.1358308307826519, "epoch": 0.32480091012514223, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022118572145700455, "kl": 0.010835035820491612, "learning_rate": 6.081911262798635e-06, "loss": -0.0117, "num_tokens": 60761434.0, "reward": 0.7250000238418579, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.3255882263183594, "sampling/importance_sampling_ratio/mean": 0.27079352736473083, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.098278045654297, "sampling/sampling_logp_difference/mean": 0.015013179741799831, "step": 571, "step_time": 74.06402071635239 }, { "clip_ratio/high_max": 0.01263001502957195, "clip_ratio/high_mean": 0.003765446977922693, "clip_ratio/low_mean": 0.0017404361642547883, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005505883164005354, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 879.7999877929688, "completions/mean_terminated_length": 639.4666748046875, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "entropy": 0.1532855499535799, "epoch": 0.3253697383390216, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013557947240769863, "kl": 0.011282958090305328, "learning_rate": 6.07679180887372e-06, "loss": -0.0408, "num_tokens": 60867482.0, "reward": 0.699999988079071, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.792679786682129, "sampling/importance_sampling_ratio/mean": 0.3831198215484619, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.844374179840088, "sampling/sampling_logp_difference/mean": 0.01556087564677, "step": 572, "step_time": 75.3285888270475 }, { "clip_ratio/high_max": 0.006429275905247778, "clip_ratio/high_mean": 0.001662109381868504, "clip_ratio/low_mean": 0.001147460952779511, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002809570389217697, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 869.4000244140625, "completions/mean_terminated_length": 611.7333374023438, "completions/min_length": 438.0, "completions/min_terminated_length": 438.0, "entropy": 0.09317250270396471, "epoch": 0.32593856655290104, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.008941888809204102, "kl": 0.011951014399528503, "learning_rate": 6.071672354948806e-06, "loss": 0.0204, "num_tokens": 60971794.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.4966306686401367, "sampling/importance_sampling_ratio/mean": 0.4282771050930023, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7398185729980469, "sampling/sampling_logp_difference/mean": 0.01194838434457779, "step": 573, "step_time": 71.77060506702401 }, { "clip_ratio/high_max": 0.011809421121142805, "clip_ratio/high_mean": 0.0036214658030075952, "clip_ratio/low_mean": 0.0038574217905988917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007478887622710317, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 879.5499877929688, "completions/mean_terminated_length": 611.2857666015625, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.13734106998890638, "epoch": 0.3265073947667804, "frac_reward_zero_std": 0.0, "grad_norm": 0.02878350019454956, "kl": 0.013143756776116788, "learning_rate": 6.066552901023891e-06, "loss": -0.0267, "num_tokens": 61080152.0, "reward": 0.675000011920929, "reward_std": 0.9442809820175171, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.216294288635254, "sampling/importance_sampling_ratio/mean": 0.38155412673950195, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6745936870574951, "sampling/sampling_logp_difference/mean": 0.015087622217833996, "step": 574, "step_time": 73.16331753297709 }, { "clip_ratio/high_max": 0.01436637487495318, "clip_ratio/high_mean": 0.00477589280490065, "clip_ratio/low_mean": 0.0025146484294964466, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0072905413107946515, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 870.6749877929688, "completions/mean_terminated_length": 663.2352905273438, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.10754358489066362, "epoch": 0.32707622298065986, "frac_reward_zero_std": 0.0, "grad_norm": 0.030885741114616394, "kl": 0.011096318950876594, "learning_rate": 6.061433447098976e-06, "loss": 0.1355, "num_tokens": 61186995.0, "reward": 0.7250000238418579, "reward_std": 0.905467689037323, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.625521421432495, "sampling/importance_sampling_ratio/mean": 0.34297364950180054, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9633373022079468, "sampling/sampling_logp_difference/mean": 0.012979275546967983, "step": 575, "step_time": 72.20597498305142 }, { "clip_ratio/high_max": 0.009104158554691821, "clip_ratio/high_mean": 0.003157397295581177, "clip_ratio/low_mean": 0.0020019531511934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005159350374015048, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 941.0, "completions/mean_length": 907.1749877929688, "completions/mean_terminated_length": 634.5833740234375, "completions/min_length": 438.0, "completions/min_terminated_length": 438.0, "entropy": 0.07158156111836433, "epoch": 0.32764505119453924, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019593684002757072, "kl": 0.00943178846500814, "learning_rate": 6.056313993174061e-06, "loss": 0.0639, "num_tokens": 61292842.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.7060141563415527, "sampling/importance_sampling_ratio/mean": 0.5416945815086365, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8630211353302002, "sampling/sampling_logp_difference/mean": 0.010175291448831558, "step": 576, "step_time": 71.88306436897255 }, { "clip_ratio/high_max": 0.012413430056767538, "clip_ratio/high_mean": 0.003839976998278871, "clip_ratio/low_mean": 0.0011564348460524343, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004996411837055348, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 942.0, "completions/mean_length": 862.4750366210938, "completions/mean_terminated_length": 665.0555419921875, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.14721585996448994, "epoch": 0.3282138794084187, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03472574055194855, "kl": 0.012055136379785836, "learning_rate": 6.051194539249147e-06, "loss": 0.1279, "num_tokens": 61399709.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.094305992126465, "sampling/importance_sampling_ratio/mean": 0.4923020303249359, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7471187114715576, "sampling/sampling_logp_difference/mean": 0.015271442010998726, "step": 577, "step_time": 71.8103365059942 }, { "clip_ratio/high_max": 0.01089074717310723, "clip_ratio/high_mean": 0.003186768932209816, "clip_ratio/low_mean": 0.002262504094687756, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005449272983241826, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 842.0, "completions/mean_length": 843.4500122070312, "completions/mean_terminated_length": 599.1764526367188, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.11266292911022902, "epoch": 0.32878270762229805, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.023800324648618698, "kl": 0.012656979088205844, "learning_rate": 6.046075085324233e-06, "loss": -0.0158, "num_tokens": 61504359.0, "reward": 0.699999988079071, "reward_std": 0.8828856945037842, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.387096643447876, "sampling/importance_sampling_ratio/mean": 0.6454517841339111, "sampling/importance_sampling_ratio/min": 0.015548239462077618, "sampling/sampling_logp_difference/max": 2.799016237258911, "sampling/sampling_logp_difference/mean": 0.01309962011873722, "step": 578, "step_time": 74.19238891382702 }, { "clip_ratio/high_max": 0.010626053786836565, "clip_ratio/high_mean": 0.003343182397657074, "clip_ratio/low_mean": 0.002465820303768851, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.0058090026141144335, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 908.1749877929688, "completions/mean_terminated_length": 602.8181762695312, "completions/min_length": 466.0, "completions/min_terminated_length": 466.0, "entropy": 0.07392575778067112, "epoch": 0.3293515358361775, "frac_reward_zero_std": 0.0, "grad_norm": 0.06807135790586472, "kl": 0.02184335025958717, "learning_rate": 6.0409556313993175e-06, "loss": -0.1028, "num_tokens": 61607718.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.561009645462036, "sampling/importance_sampling_ratio/mean": 0.48599568009376526, "sampling/importance_sampling_ratio/min": 0.0006772018386982381, "sampling/sampling_logp_difference/max": 5.721366882324219, "sampling/sampling_logp_difference/mean": 0.010372772812843323, "step": 579, "step_time": 72.9853176374454 }, { "clip_ratio/high_max": 0.016799110104329884, "clip_ratio/high_mean": 0.006680279708234593, "clip_ratio/low_mean": 0.0010253906148136593, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007705670257564634, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 803.0, "completions/mean_length": 597.7750244140625, "completions/mean_terminated_length": 455.70001220703125, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 0.11395192984491587, "epoch": 0.32992036405005687, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04084199666976929, "kl": 0.0195426078280434, "learning_rate": 6.035836177474403e-06, "loss": -0.1397, "num_tokens": 61699565.0, "reward": 1.5, "reward_std": 0.8770580887794495, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.75, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.7357161045074463, "sampling/importance_sampling_ratio/mean": 0.5931078195571899, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5086939334869385, "sampling/sampling_logp_difference/mean": 0.015652699396014214, "step": 580, "step_time": 70.32340337778442 }, { "clip_ratio/high_max": 0.012916840089019388, "clip_ratio/high_mean": 0.003424171212827787, "clip_ratio/low_mean": 0.0006550339021487162, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004079205129528418, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 867.0, "completions/mean_length": 883.0250244140625, "completions/mean_terminated_length": 554.0833740234375, "completions/min_length": 386.0, "completions/min_terminated_length": 386.0, "entropy": 0.11914434563368559, "epoch": 0.3304891922639363, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.014710037969052792, "kl": 0.017151751555502415, "learning_rate": 6.030716723549488e-06, "loss": -0.0257, "num_tokens": 61814990.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.7750630378723145, "sampling/importance_sampling_ratio/mean": 0.3542335629463196, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8778036832809448, "sampling/sampling_logp_difference/mean": 0.013493726961314678, "step": 581, "step_time": 76.28994824481197 }, { "clip_ratio/high_max": 0.01491652918048203, "clip_ratio/high_mean": 0.004120377081562765, "clip_ratio/low_mean": 0.001274523785468773, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005394900916144252, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 707.5, "completions/mean_terminated_length": 571.857177734375, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.12909972108900547, "epoch": 0.3310580204778157, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022715702652931213, "kl": 0.01247302140109241, "learning_rate": 6.025597269624574e-06, "loss": 0.0491, "num_tokens": 61907034.0, "reward": 1.375, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.496091604232788, "sampling/importance_sampling_ratio/mean": 0.4315655827522278, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7132554054260254, "sampling/sampling_logp_difference/mean": 0.014716743491590023, "step": 582, "step_time": 66.73421693663113 }, { "clip_ratio/high_max": 0.005875220813322812, "clip_ratio/high_mean": 0.0013048468681517988, "clip_ratio/low_mean": 0.0019124513491988182, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032172982464544475, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 963.0, "completions/mean_length": 898.875, "completions/mean_terminated_length": 639.0, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.07779250433668494, "epoch": 0.3316268486916951, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025556934997439384, "kl": 0.010860820126254112, "learning_rate": 6.020477815699658e-06, "loss": -0.0236, "num_tokens": 62010685.0, "reward": 0.42500001192092896, "reward_std": 0.6751068830490112, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.922452926635742, "sampling/importance_sampling_ratio/mean": 0.6460593938827515, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.621781826019287, "sampling/sampling_logp_difference/mean": 0.010661371983587742, "step": 583, "step_time": 72.69570663711056 }, { "clip_ratio/high_max": 0.006282517744693905, "clip_ratio/high_mean": 0.0012565035722218454, "clip_ratio/low_mean": 0.0008789062594587449, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002135409762558993, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 836.0, "completions/mean_length": 977.9249877929688, "completions/mean_terminated_length": 655.4000244140625, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.09958312660455704, "epoch": 0.3321956769055745, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.016718944534659386, "kl": 0.009237674588803202, "learning_rate": 6.015358361774744e-06, "loss": 0.0244, "num_tokens": 62121842.0, "reward": 0.25, "reward_std": 0.6698641180992126, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 1.4105496406555176, "sampling/importance_sampling_ratio/mean": 0.40976208448410034, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9747779369354248, "sampling/sampling_logp_difference/mean": 0.011283197440207005, "step": 584, "step_time": 74.17727458011359 }, { "clip_ratio/high_max": 0.013722509262152016, "clip_ratio/high_mean": 0.004618198494426906, "clip_ratio/low_mean": 0.0021240234418655746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006742221914464608, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 843.7250366210938, "completions/mean_terminated_length": 543.2667236328125, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.12067434843629599, "epoch": 0.33276450511945393, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027073975652456284, "kl": 0.01139134808909148, "learning_rate": 6.010238907849829e-06, "loss": -0.1388, "num_tokens": 62226223.0, "reward": 0.675000011920929, "reward_std": 0.9167249202728271, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.3026907444000244, "sampling/importance_sampling_ratio/mean": 0.42578715085983276, "sampling/importance_sampling_ratio/min": 0.0021710279397666454, "sampling/sampling_logp_difference/max": 1.7358307838439941, "sampling/sampling_logp_difference/mean": 0.014107334427535534, "step": 585, "step_time": 72.18659315095283 }, { "clip_ratio/high_max": 0.009846414846833795, "clip_ratio/high_mean": 0.003577648152713664, "clip_ratio/low_mean": 0.0029823955992469564, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.006560043984791264, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 790.7999877929688, "completions/mean_terminated_length": 557.6000366210938, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.08949874062091112, "epoch": 0.3333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 0.04075933247804642, "kl": 0.013782298541627824, "learning_rate": 6.0051194539249156e-06, "loss": -0.1012, "num_tokens": 62338679.0, "reward": 0.8500000238418579, "reward_std": 0.9212589263916016, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.962486743927002, "sampling/importance_sampling_ratio/mean": 0.3104127049446106, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2712066173553467, "sampling/sampling_logp_difference/mean": 0.01265606191009283, "step": 586, "step_time": 76.83945465111174 }, { "clip_ratio/high_max": 0.01203384983818978, "clip_ratio/high_mean": 0.004398792778374627, "clip_ratio/low_mean": 0.002068284127744846, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006467076775152236, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 799.4500122070312, "completions/mean_terminated_length": 714.27587890625, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.16270972788333893, "epoch": 0.33390216154721275, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.035911235958337784, "kl": 0.011817097081802785, "learning_rate": 6e-06, "loss": 0.0876, "num_tokens": 62446777.0, "reward": 1.3000000715255737, "reward_std": 0.8828856945037842, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.620253801345825, "sampling/importance_sampling_ratio/mean": 0.6369496583938599, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.027479648590088, "sampling/sampling_logp_difference/mean": 0.0171852707862854, "step": 587, "step_time": 72.40663886675611 }, { "clip_ratio/high_max": 0.014081605593673885, "clip_ratio/high_mean": 0.005640203948132694, "clip_ratio/low_mean": 0.002368164059589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00800836825510487, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 771.375, "completions/mean_terminated_length": 564.6818237304688, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.13806413859128952, "epoch": 0.33447098976109213, "frac_reward_zero_std": 0.0, "grad_norm": 0.02919682487845421, "kl": 0.017550822347402573, "learning_rate": 5.994880546075086e-06, "loss": 0.1176, "num_tokens": 62556104.0, "reward": 1.0500000715255737, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.423170804977417, "sampling/importance_sampling_ratio/mean": 0.3700469136238098, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.392911911010742, "sampling/sampling_logp_difference/mean": 0.01577596366405487, "step": 588, "step_time": 75.69905938114971 }, { "clip_ratio/high_max": 0.01305015932302922, "clip_ratio/high_mean": 0.00411969333072193, "clip_ratio/low_mean": 0.0009033203150465852, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005023013713071123, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 792.0, "completions/mean_length": 870.125, "completions/mean_terminated_length": 613.6666870117188, "completions/min_length": 457.0, "completions/min_terminated_length": 457.0, "entropy": 0.07710640225559473, "epoch": 0.33503981797497157, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 4.046716690063477, "kl": 0.599889507109765, "learning_rate": 5.989761092150171e-06, "loss": -0.0535, "num_tokens": 62665981.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.1871602535247803, "sampling/importance_sampling_ratio/mean": 0.458395779132843, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.923225402832031, "sampling/sampling_logp_difference/mean": 0.01133275032043457, "step": 589, "step_time": 74.7568652483169 }, { "clip_ratio/high_max": 0.008170349290594459, "clip_ratio/high_mean": 0.0017758468020474538, "clip_ratio/low_mean": 0.0006591796991415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024350265157409012, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 904.6749877929688, "completions/mean_terminated_length": 725.6875, "completions/min_length": 433.0, "completions/min_terminated_length": 433.0, "entropy": 0.05820251861587167, "epoch": 0.33560864618885095, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.014593382366001606, "kl": 0.017287515220232308, "learning_rate": 5.9846416382252565e-06, "loss": 0.0496, "num_tokens": 62752608.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.6389153003692627, "sampling/importance_sampling_ratio/mean": 0.6033217906951904, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.216421604156494, "sampling/sampling_logp_difference/mean": 0.008148341439664364, "step": 590, "step_time": 69.75847205379978 }, { "clip_ratio/high_max": 0.011372967856004834, "clip_ratio/high_mean": 0.005259822239167988, "clip_ratio/low_mean": 0.0030517578197759576, "clip_ratio/low_min": 0.000732421875, "clip_ratio/region_mean": 0.00831158016808331, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 974.0, "completions/mean_length": 865.1749877929688, "completions/mean_terminated_length": 650.2941284179688, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.1663778554648161, "epoch": 0.3361774744027304, "frac_reward_zero_std": 0.0, "grad_norm": 0.06169770658016205, "kl": 0.01596154214348644, "learning_rate": 5.979522184300341e-06, "loss": -0.1664, "num_tokens": 62864583.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.758333683013916, "sampling/importance_sampling_ratio/mean": 0.45580750703811646, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0506069660186768, "sampling/sampling_logp_difference/mean": 0.016606658697128296, "step": 591, "step_time": 82.78838860453106 }, { "clip_ratio/high_max": 0.006774961308110505, "clip_ratio/high_mean": 0.0026154085499001667, "clip_ratio/low_mean": 0.002490234335709829, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005105642921989784, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 911.2000122070312, "completions/mean_terminated_length": 613.8181762695312, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.0979261938482523, "epoch": 0.33674630261660976, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01548747904598713, "kl": 0.013989464147016406, "learning_rate": 5.9744027303754265e-06, "loss": 0.016, "num_tokens": 62974295.0, "reward": 0.4749999940395355, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.1800694465637207, "sampling/importance_sampling_ratio/mean": 0.22486528754234314, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3052797317504883, "sampling/sampling_logp_difference/mean": 0.011719108559191227, "step": 592, "step_time": 75.01214362611063 }, { "clip_ratio/high_max": 0.010641483822837472, "clip_ratio/high_mean": 0.0024204343935707584, "clip_ratio/low_mean": 0.003267311694798991, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005687745986506343, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 883.7750244140625, "completions/mean_terminated_length": 556.5833740234375, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.16598735749721527, "epoch": 0.3373151308304892, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.06580831855535507, "kl": 0.014401905005797744, "learning_rate": 5.969283276450512e-06, "loss": -0.2231, "num_tokens": 63087974.0, "reward": 0.4749999940395355, "reward_std": 0.7840558886528015, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.980602264404297, "sampling/importance_sampling_ratio/mean": 0.4804622530937195, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.688424587249756, "sampling/sampling_logp_difference/mean": 0.016857005655765533, "step": 593, "step_time": 73.6304097129032 }, { "clip_ratio/high_max": 0.011894802446477115, "clip_ratio/high_mean": 0.0024676129542058334, "clip_ratio/low_mean": 0.0020263672049622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004493980202823877, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.0, "completions/mean_length": 844.9750366210938, "completions/mean_terminated_length": 576.4375, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "entropy": 0.11091499123722315, "epoch": 0.3378839590443686, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01031448133289814, "kl": 0.012766001047566533, "learning_rate": 5.964163822525598e-06, "loss": 0.018, "num_tokens": 63194789.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.230229616165161, "sampling/importance_sampling_ratio/mean": 0.296372652053833, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.203464984893799, "sampling/sampling_logp_difference/mean": 0.013500970788300037, "step": 594, "step_time": 74.50292942486703 }, { "clip_ratio/high_max": 0.01484180276747793, "clip_ratio/high_mean": 0.004703876504208893, "clip_ratio/low_mean": 0.0006591796991415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005363056159694679, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 891.1749877929688, "completions/mean_terminated_length": 615.3077392578125, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "entropy": 0.12542272731661797, "epoch": 0.338452787258248, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01819830946624279, "kl": 0.011293853516690433, "learning_rate": 5.959044368600683e-06, "loss": -0.0077, "num_tokens": 63306892.0, "reward": 0.6000000238418579, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.7826082706451416, "sampling/importance_sampling_ratio/mean": 0.3965797424316406, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8193691968917847, "sampling/sampling_logp_difference/mean": 0.013767621479928493, "step": 595, "step_time": 75.25918865483254 }, { "clip_ratio/high_max": 0.01278376707341522, "clip_ratio/high_mean": 0.005276983618387021, "clip_ratio/low_mean": 0.0018066406264551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0070836240774951875, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 860.9500122070312, "completions/mean_terminated_length": 752.25, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.09742054436355829, "epoch": 0.3390216154721274, "frac_reward_zero_std": 0.0, "grad_norm": 0.018178772181272507, "kl": 0.014336429303511977, "learning_rate": 5.953924914675768e-06, "loss": -0.0139, "num_tokens": 63407602.0, "reward": 0.9750000238418579, "reward_std": 0.8911962509155273, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.5512917041778564, "sampling/importance_sampling_ratio/mean": 0.29426321387290955, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.593991279602051, "sampling/sampling_logp_difference/mean": 0.012144550681114197, "step": 596, "step_time": 73.44540412607603 }, { "clip_ratio/high_max": 0.013907274464145303, "clip_ratio/high_mean": 0.005073310967418365, "clip_ratio/low_mean": 0.001953125012732926, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007026436098385602, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 851.6749877929688, "completions/mean_terminated_length": 618.5294189453125, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.08797485660761595, "epoch": 0.3395904436860068, "frac_reward_zero_std": 0.0, "grad_norm": 0.028905009850859642, "kl": 0.011202819412574172, "learning_rate": 5.948805460750854e-06, "loss": -0.0531, "num_tokens": 63512773.0, "reward": 0.800000011920929, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.9271621704101562, "sampling/importance_sampling_ratio/mean": 0.456550270318985, "sampling/importance_sampling_ratio/min": 0.00508519122377038, "sampling/sampling_logp_difference/max": 2.9676551818847656, "sampling/sampling_logp_difference/mean": 0.011305016465485096, "step": 597, "step_time": 71.9806959235575 }, { "clip_ratio/high_max": 0.01373741909628734, "clip_ratio/high_mean": 0.0066800196073018014, "clip_ratio/low_mean": 0.0016113281453726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008291347738122568, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 737.375, "completions/mean_terminated_length": 546.2916870117188, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.10240485053509474, "epoch": 0.34015927189988626, "frac_reward_zero_std": 0.0, "grad_norm": 0.0628352090716362, "kl": 0.01367159083019942, "learning_rate": 5.943686006825939e-06, "loss": 0.0604, "num_tokens": 63614948.0, "reward": 1.1750000715255737, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.1941704750061035, "sampling/importance_sampling_ratio/mean": 0.4904846251010895, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.555642604827881, "sampling/sampling_logp_difference/mean": 0.012992987409234047, "step": 598, "step_time": 70.81964104692452 }, { "clip_ratio/high_max": 0.010313626320566982, "clip_ratio/high_mean": 0.002807871933327988, "clip_ratio/low_mean": 0.002026367175858468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004834239138290286, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 884.9249877929688, "completions/mean_terminated_length": 560.4166870117188, "completions/min_length": 418.0, "completions/min_terminated_length": 418.0, "entropy": 0.12632365431636572, "epoch": 0.34072810011376564, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.057579051703214645, "kl": 0.01191557792481035, "learning_rate": 5.938566552901024e-06, "loss": -0.1958, "num_tokens": 63724465.0, "reward": 0.574999988079071, "reward_std": 0.9026315212249756, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.822413682937622, "sampling/importance_sampling_ratio/mean": 0.3850945234298706, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.555530548095703, "sampling/sampling_logp_difference/mean": 0.01432985719293356, "step": 599, "step_time": 74.41793642891571 }, { "clip_ratio/high_max": 0.006715128198266029, "clip_ratio/high_mean": 0.002187488629715517, "clip_ratio/low_mean": 0.0009136583030340262, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003101146918197628, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 956.0, "completions/mean_length": 918.4000244140625, "completions/mean_terminated_length": 672.0, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.11174089647829533, "epoch": 0.3412969283276451, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01107555441558361, "kl": 0.01566179422661662, "learning_rate": 5.933447098976109e-06, "loss": 0.0177, "num_tokens": 63836377.0, "reward": 0.4000000059604645, "reward_std": 0.6717752814292908, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 1.821658730506897, "sampling/importance_sampling_ratio/mean": 0.3347683846950531, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.6882224082946777, "sampling/sampling_logp_difference/mean": 0.013334422372281551, "step": 600, "step_time": 76.85860466887243 }, { "clip_ratio/high_max": 0.00984688347671181, "clip_ratio/high_mean": 0.002534791565267369, "clip_ratio/low_mean": 0.001843440728407586, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004378232290036976, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 923.375, "completions/mean_terminated_length": 576.7777709960938, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.10205286368727684, "epoch": 0.34186575654152446, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03483866900205612, "kl": 0.009707012097351253, "learning_rate": 5.9283276450511945e-06, "loss": -0.032, "num_tokens": 63949640.0, "reward": 0.42500001192092896, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.341810941696167, "sampling/importance_sampling_ratio/mean": 0.46074748039245605, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.002777576446533, "sampling/sampling_logp_difference/mean": 0.011107824742794037, "step": 601, "step_time": 76.60438160225749 }, { "clip_ratio/high_max": 0.012790455075446516, "clip_ratio/high_mean": 0.0037140567146707326, "clip_ratio/low_mean": 0.0016737364894652274, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00538779329508543, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.0, "completions/mean_length": 695.7999877929688, "completions/mean_terminated_length": 537.7777709960938, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0886025745421648, "epoch": 0.3424345847554039, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030808649957180023, "kl": 0.011714725114870816, "learning_rate": 5.923208191126279e-06, "loss": -0.0821, "num_tokens": 64044544.0, "reward": 1.149999976158142, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.4639437198638916, "sampling/importance_sampling_ratio/mean": 0.414838969707489, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.618962526321411, "sampling/sampling_logp_difference/mean": 0.012797883711755276, "step": 602, "step_time": 71.09220296610147 }, { "clip_ratio/high_max": 0.015655759605579078, "clip_ratio/high_mean": 0.004709318105597049, "clip_ratio/low_mean": 0.002001953122089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00671127118403092, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 950.0, "completions/mean_length": 750.875, "completions/mean_terminated_length": 527.4091186523438, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.10831670742481947, "epoch": 0.3430034129692833, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02240528166294098, "kl": 0.01466565893497318, "learning_rate": 5.918088737201365e-06, "loss": -0.0365, "num_tokens": 64147563.0, "reward": 1.0750000476837158, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.170898199081421, "sampling/importance_sampling_ratio/mean": 0.47918501496315, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4662973880767822, "sampling/sampling_logp_difference/mean": 0.014185707084834576, "step": 603, "step_time": 73.2899352258537 }, { "clip_ratio/high_max": 0.014748568646609783, "clip_ratio/high_mean": 0.006782633674447425, "clip_ratio/low_mean": 0.0023386287448374787, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009121262177359313, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 990.0, "completions/mean_length": 708.6000366210938, "completions/mean_terminated_length": 573.4285888671875, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.1533445343375206, "epoch": 0.3435722411831627, "frac_reward_zero_std": 0.0, "grad_norm": 0.046136029064655304, "kl": 0.013035411830060184, "learning_rate": 5.912969283276451e-06, "loss": 0.0477, "num_tokens": 64246155.0, "reward": 1.274999976158142, "reward_std": 0.905467689037323, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.9386465549468994, "sampling/importance_sampling_ratio/mean": 0.5307098627090454, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6463186740875244, "sampling/sampling_logp_difference/mean": 0.01717379502952099, "step": 604, "step_time": 69.87001130636781 }, { "clip_ratio/high_max": 0.00407880824059248, "clip_ratio/high_mean": 0.0011474376224214211, "clip_ratio/low_mean": 0.0010498046867724042, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021972423091938253, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 871.0, "completions/mean_length": 802.875, "completions/mean_terminated_length": 621.95458984375, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.09749812912195921, "epoch": 0.3441410693970421, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01409077923744917, "kl": 0.013308501685969532, "learning_rate": 5.907849829351536e-06, "loss": -0.0123, "num_tokens": 64353254.0, "reward": 1.100000023841858, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.7657806873321533, "sampling/importance_sampling_ratio/mean": 0.328984797000885, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4693422317504883, "sampling/sampling_logp_difference/mean": 0.013194783590734005, "step": 605, "step_time": 73.46035003266297 }, { "clip_ratio/high_max": 0.010675654921215028, "clip_ratio/high_mean": 0.00291555943840649, "clip_ratio/low_mean": 0.001489257840148639, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004404817271279171, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 752.0250244140625, "completions/mean_terminated_length": 621.0740966796875, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.11922928411513567, "epoch": 0.3447098976109215, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016280055046081543, "kl": 0.014365371665917337, "learning_rate": 5.902730375426621e-06, "loss": -0.0007, "num_tokens": 64466183.0, "reward": 1.3000000715255737, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.3991520404815674, "sampling/importance_sampling_ratio/mean": 0.3864131569862366, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.551255464553833, "sampling/sampling_logp_difference/mean": 0.015408082865178585, "step": 606, "step_time": 74.3722166295629 }, { "clip_ratio/high_max": 0.010026624833699316, "clip_ratio/high_mean": 0.002005324946367182, "clip_ratio/low_mean": 0.002221679653302999, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004227004654239863, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 959.3250122070312, "completions/mean_terminated_length": 736.5555419921875, "completions/min_length": 467.0, "completions/min_terminated_length": 467.0, "entropy": 0.14832168258726597, "epoch": 0.3452787258248009, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0164786484092474, "kl": 0.012292700936086476, "learning_rate": 5.897610921501706e-06, "loss": 0.0395, "num_tokens": 64580316.0, "reward": 0.4000000059604645, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.8957624435424805, "sampling/importance_sampling_ratio/mean": 0.3304823040962219, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3264784812927246, "sampling/sampling_logp_difference/mean": 0.01568092219531536, "step": 607, "step_time": 76.61763457907364 }, { "clip_ratio/high_max": 0.010670856456272304, "clip_ratio/high_mean": 0.003251522153732367, "clip_ratio/low_mean": 0.0009571792143105995, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004208701429888606, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 876.0, "completions/mean_length": 820.5250244140625, "completions/mean_terminated_length": 595.631591796875, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.10119823087006807, "epoch": 0.34584755403868034, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.028953388333320618, "kl": 0.010144805419258773, "learning_rate": 5.892491467576792e-06, "loss": 0.1391, "num_tokens": 64687145.0, "reward": 0.75, "reward_std": 0.8987169861793518, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.8760037422180176, "sampling/importance_sampling_ratio/mean": 0.5005548596382141, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9948384761810303, "sampling/sampling_logp_difference/mean": 0.011757249012589455, "step": 608, "step_time": 72.559381891042 }, { "clip_ratio/high_max": 0.008381780004128814, "clip_ratio/high_mean": 0.003087218792643398, "clip_ratio/low_mean": 0.00292417174205184, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006011390476487577, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 939.0, "completions/mean_length": 920.125, "completions/mean_terminated_length": 677.75, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.12965272273868322, "epoch": 0.3464163822525597, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.016706114634871483, "kl": 0.011620408913586289, "learning_rate": 5.887372013651878e-06, "loss": 0.0931, "num_tokens": 64794382.0, "reward": 0.4749999940395355, "reward_std": 0.7840558886528015, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.8686134815216064, "sampling/importance_sampling_ratio/mean": 0.3211049735546112, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.880473256111145, "sampling/sampling_logp_difference/mean": 0.01480275671929121, "step": 609, "step_time": 73.03523359168321 }, { "clip_ratio/high_max": 0.013754119980148971, "clip_ratio/high_mean": 0.004212293744785711, "clip_ratio/low_mean": 0.0020145363232586533, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006226830038940534, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 777.0, "completions/mean_length": 765.5250244140625, "completions/mean_terminated_length": 531.6666870117188, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.10529939737170935, "epoch": 0.34698521046643915, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.035041388124227524, "kl": 0.012165450607426465, "learning_rate": 5.882252559726963e-06, "loss": -0.016, "num_tokens": 64885107.0, "reward": 0.949999988079071, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.4037044048309326, "sampling/importance_sampling_ratio/mean": 0.495988130569458, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.107125759124756, "sampling/sampling_logp_difference/mean": 0.013814064674079418, "step": 610, "step_time": 71.14031734783202 }, { "clip_ratio/high_max": 0.014156623801682144, "clip_ratio/high_mean": 0.0051270363765070215, "clip_ratio/low_mean": 0.0006591796991415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00578621614840813, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 689.25, "completions/mean_terminated_length": 488.3999938964844, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.09536533057689667, "epoch": 0.34755403868031853, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.05780467018485069, "kl": 0.012338222120888531, "learning_rate": 5.877133105802048e-06, "loss": 0.3208, "num_tokens": 64989989.0, "reward": 1.25, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.7830770015716553, "sampling/importance_sampling_ratio/mean": 0.5922564268112183, "sampling/importance_sampling_ratio/min": 0.011231712065637112, "sampling/sampling_logp_difference/max": 2.0026779174804688, "sampling/sampling_logp_difference/mean": 0.013154144398868084, "step": 611, "step_time": 73.47028220747598 }, { "clip_ratio/high_max": 0.006959980819374323, "clip_ratio/high_mean": 0.002169065162888728, "clip_ratio/low_mean": 0.0016845702994032763, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038536354404641315, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 861.7999877929688, "completions/mean_terminated_length": 618.5, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.10038431826978922, "epoch": 0.34812286689419797, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02985323593020439, "kl": 0.012776549090631306, "learning_rate": 5.8720136518771335e-06, "loss": -0.0222, "num_tokens": 65099317.0, "reward": 0.800000011920929, "reward_std": 0.9922778606414795, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.6674840450286865, "sampling/importance_sampling_ratio/mean": 0.3264191746711731, "sampling/importance_sampling_ratio/min": 0.0014301426708698273, "sampling/sampling_logp_difference/max": 3.2464256286621094, "sampling/sampling_logp_difference/mean": 0.013446033000946045, "step": 612, "step_time": 72.21294781868346 }, { "clip_ratio/high_max": 0.013083076453767717, "clip_ratio/high_mean": 0.0034729142498690635, "clip_ratio/low_mean": 0.0028320312267169356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0063049455638974905, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 842.7000122070312, "completions/mean_terminated_length": 642.3157958984375, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.13109242357313633, "epoch": 0.34869169510807735, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02735080011188984, "kl": 0.015818202984519303, "learning_rate": 5.866894197952219e-06, "loss": -0.03, "num_tokens": 65211393.0, "reward": 0.9000000357627869, "reward_std": 0.9818872213363647, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.091768980026245, "sampling/importance_sampling_ratio/mean": 0.43110305070877075, "sampling/importance_sampling_ratio/min": 0.00022417728905566037, "sampling/sampling_logp_difference/max": 4.6199750900268555, "sampling/sampling_logp_difference/mean": 0.016019513830542564, "step": 613, "step_time": 75.84769438998774 }, { "clip_ratio/high_max": 0.013527431874535978, "clip_ratio/high_mean": 0.00446926690347027, "clip_ratio/low_mean": 0.002294921847351361, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006764188874512911, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 953.0, "completions/mean_length": 850.7999877929688, "completions/mean_terminated_length": 659.368408203125, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.10536645352840424, "epoch": 0.3492605233219568, "frac_reward_zero_std": 0.0, "grad_norm": 0.017856042832136154, "kl": 0.015196236956398934, "learning_rate": 5.8617747440273035e-06, "loss": -0.0585, "num_tokens": 65318641.0, "reward": 0.9000000357627869, "reward_std": 0.9818872213363647, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.4374215602874756, "sampling/importance_sampling_ratio/mean": 0.3102102279663086, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6983859539031982, "sampling/sampling_logp_difference/mean": 0.013573059812188148, "step": 614, "step_time": 74.14577515865676 }, { "clip_ratio/high_max": 0.011146775097586215, "clip_ratio/high_mean": 0.00411290860211011, "clip_ratio/low_mean": 0.001196289056679234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00530919770244509, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 888.0250244140625, "completions/mean_terminated_length": 684.0625, "completions/min_length": 377.0, "completions/min_terminated_length": 377.0, "entropy": 0.13344868458807468, "epoch": 0.34982935153583616, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.035804156213998795, "kl": 0.014256106573157012, "learning_rate": 5.856655290102389e-06, "loss": 0.1252, "num_tokens": 65426850.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.2851696014404297, "sampling/importance_sampling_ratio/mean": 0.36690521240234375, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5945541858673096, "sampling/sampling_logp_difference/mean": 0.015153474174439907, "step": 615, "step_time": 74.33844787371345 }, { "clip_ratio/high_max": 0.011004537693224847, "clip_ratio/high_mean": 0.0026733954873634502, "clip_ratio/low_mean": 0.001733398465148639, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004406793974339962, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 855.0, "completions/mean_length": 806.6000366210938, "completions/mean_terminated_length": 512.4705810546875, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.11624537408351898, "epoch": 0.3503981797497156, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03496754541993141, "kl": 0.009706664306577295, "learning_rate": 5.851535836177474e-06, "loss": -0.0692, "num_tokens": 65532098.0, "reward": 0.7750000357627869, "reward_std": 0.9469925761222839, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.9197232723236084, "sampling/importance_sampling_ratio/mean": 0.5382119417190552, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.979138731956482, "sampling/sampling_logp_difference/mean": 0.013714211992919445, "step": 616, "step_time": 71.61419808445498 }, { "clip_ratio/high_max": 0.012447372020687908, "clip_ratio/high_mean": 0.0035806515952572227, "clip_ratio/low_mean": 0.0022460937507275958, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005826745327794924, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.0, "completions/mean_length": 895.2999877929688, "completions/mean_terminated_length": 702.25, "completions/min_length": 427.0, "completions/min_terminated_length": 427.0, "entropy": 0.08183241169899702, "epoch": 0.350967007963595, "frac_reward_zero_std": 0.0, "grad_norm": 0.020420320332050323, "kl": 0.010038523061666638, "learning_rate": 5.846416382252561e-06, "loss": -0.0109, "num_tokens": 65644278.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.97684907913208, "sampling/importance_sampling_ratio/mean": 0.3580781817436218, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.9561827182769775, "sampling/sampling_logp_difference/mean": 0.01129829790443182, "step": 617, "step_time": 74.97740031499416 }, { "clip_ratio/high_max": 0.013807471201289445, "clip_ratio/high_mean": 0.004648736648960039, "clip_ratio/low_mean": 0.001245117196958745, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005893853842280805, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 679.6749877929688, "completions/mean_terminated_length": 593.59375, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.15413894038647413, "epoch": 0.3515358361774744, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.00990383978933096, "kl": 0.017054351163096726, "learning_rate": 5.841296928327645e-06, "loss": -0.0159, "num_tokens": 65745585.0, "reward": 1.600000023841858, "reward_std": 0.8101914525032043, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.800000011920929, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.355541467666626, "sampling/importance_sampling_ratio/mean": 0.4863249361515045, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4621007442474365, "sampling/sampling_logp_difference/mean": 0.01842012070119381, "step": 618, "step_time": 71.17703388421796 }, { "clip_ratio/high_max": 0.009596528369002044, "clip_ratio/high_mean": 0.002710025670239702, "clip_ratio/low_mean": 0.0019287109607830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004638736485503614, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 958.1749877929688, "completions/mean_terminated_length": 760.7000122070312, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.13165869191288948, "epoch": 0.3521046643913538, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024128111079335213, "kl": 0.008917516388464719, "learning_rate": 5.836177474402731e-06, "loss": -0.0458, "num_tokens": 65851088.0, "reward": 0.5, "reward_std": 0.8770580291748047, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.398334503173828, "sampling/importance_sampling_ratio/mean": 0.3888825476169586, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4900672435760498, "sampling/sampling_logp_difference/mean": 0.013214088045060635, "step": 619, "step_time": 77.21896823076531 }, { "clip_ratio/high_max": 0.01212189788930118, "clip_ratio/high_mean": 0.004223007388645783, "clip_ratio/low_mean": 0.0010986328015860636, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005321640201145783, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 792.0, "completions/mean_length": 708.9000244140625, "completions/mean_terminated_length": 573.857177734375, "completions/min_length": 406.0, "completions/min_terminated_length": 406.0, "entropy": 0.12131457589566708, "epoch": 0.35267349260523323, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.047725122421979904, "kl": 0.01663783041294664, "learning_rate": 5.831058020477816e-06, "loss": -0.1563, "num_tokens": 65949940.0, "reward": 1.225000023841858, "reward_std": 0.8911962509155273, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.110629081726074, "sampling/importance_sampling_ratio/mean": 0.33890944719314575, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1504359245300293, "sampling/sampling_logp_difference/mean": 0.01617847941815853, "step": 620, "step_time": 71.15884386491962 }, { "clip_ratio/high_max": 0.015589785180054605, "clip_ratio/high_mean": 0.006001880596159026, "clip_ratio/low_mean": 0.0022603977704420686, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008262278221081942, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 941.0, "completions/mean_length": 819.5, "completions/mean_terminated_length": 709.3846435546875, "completions/min_length": 462.0, "completions/min_terminated_length": 462.0, "entropy": 0.1238354379311204, "epoch": 0.3532423208191126, "frac_reward_zero_std": 0.0, "grad_norm": 0.017981454730033875, "kl": 0.01647060934919864, "learning_rate": 5.8259385665529015e-06, "loss": 0.0158, "num_tokens": 66046688.0, "reward": 1.0750000476837158, "reward_std": 0.888314425945282, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.6379194259643555, "sampling/importance_sampling_ratio/mean": 0.21502844989299774, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.813467025756836, "sampling/sampling_logp_difference/mean": 0.015216889791190624, "step": 621, "step_time": 72.24214507057332 }, { "clip_ratio/high_max": 0.01435992680490017, "clip_ratio/high_mean": 0.004003915761131793, "clip_ratio/low_mean": 0.0023246046184794977, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006328520365059376, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 828.875, "completions/mean_terminated_length": 613.2105102539062, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.08298278041183949, "epoch": 0.35381114903299204, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014163434505462646, "kl": 0.011636263923719525, "learning_rate": 5.820819112627986e-06, "loss": 0.0271, "num_tokens": 66156315.0, "reward": 0.824999988079071, "reward_std": 0.9306049942970276, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.44068443775177, "sampling/importance_sampling_ratio/mean": 0.34859490394592285, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.115100383758545, "sampling/sampling_logp_difference/mean": 0.01166824996471405, "step": 622, "step_time": 75.2513353289105 }, { "clip_ratio/high_max": 0.005294897826388478, "clip_ratio/high_mean": 0.0014235882554203272, "clip_ratio/low_mean": 0.0019042969215661287, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033278852060902864, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 961.2999877929688, "completions/mean_terminated_length": 606.0, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.09357328899204731, "epoch": 0.3543799772468714, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013378937728703022, "kl": 0.013063766236882657, "learning_rate": 5.8156996587030715e-06, "loss": 0.0819, "num_tokens": 66273743.0, "reward": 0.22500000894069672, "reward_std": 0.5767948627471924, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 1.7317695617675781, "sampling/importance_sampling_ratio/mean": 0.352665513753891, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.526754140853882, "sampling/sampling_logp_difference/mean": 0.010692061856389046, "step": 623, "step_time": 76.64485457236879 }, { "clip_ratio/high_max": 0.013386808568611741, "clip_ratio/high_mean": 0.0038039945357013494, "clip_ratio/low_mean": 0.002446021870127879, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006250016303965822, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 952.0, "completions/mean_length": 903.6749877929688, "completions/mean_terminated_length": 622.9166870117188, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "entropy": 0.12211599759757519, "epoch": 0.35494880546075086, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021670633926987648, "kl": 0.014495458803139627, "learning_rate": 5.810580204778157e-06, "loss": -0.0328, "num_tokens": 66387602.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.4154715538024902, "sampling/importance_sampling_ratio/mean": 0.36834949254989624, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3494415283203125, "sampling/sampling_logp_difference/mean": 0.014365813694894314, "step": 624, "step_time": 76.85834647878073 }, { "clip_ratio/high_max": 0.016456986020784825, "clip_ratio/high_mean": 0.006389285284967627, "clip_ratio/low_mean": 0.001440429678041255, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007829714973922819, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 837.5750122070312, "completions/mean_terminated_length": 609.7222290039062, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.1112084025517106, "epoch": 0.35551763367463024, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019207578152418137, "kl": 0.015686762635596097, "learning_rate": 5.805460750853243e-06, "loss": 0.0529, "num_tokens": 66495225.0, "reward": 0.824999988079071, "reward_std": 0.957761824131012, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.1957310438156128, "sampling/importance_sampling_ratio/mean": 0.27883607149124146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3841192722320557, "sampling/sampling_logp_difference/mean": 0.014196708798408508, "step": 625, "step_time": 74.85187858226709 }, { "clip_ratio/high_max": 0.019433145876973867, "clip_ratio/high_mean": 0.005242140643531457, "clip_ratio/low_mean": 0.0032066007261164486, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008448741282336414, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 673.4000244140625, "completions/mean_terminated_length": 504.59259033203125, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.12789813056588173, "epoch": 0.3560864618885097, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029298165813088417, "kl": 0.02845529420301318, "learning_rate": 5.800341296928328e-06, "loss": 0.0946, "num_tokens": 66598681.0, "reward": 1.149999976158142, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.38032865524292, "sampling/importance_sampling_ratio/mean": 0.37547391653060913, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.569476127624512, "sampling/sampling_logp_difference/mean": 0.017695987597107887, "step": 626, "step_time": 70.41624881024472 }, { "clip_ratio/high_max": 0.009351960034109652, "clip_ratio/high_mean": 0.0030741370137548074, "clip_ratio/low_mean": 0.0012939452863065526, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0043680822709575295, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 824.7999877929688, "completions/mean_terminated_length": 644.5714111328125, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "entropy": 0.10683185700327158, "epoch": 0.35665529010238906, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.023367537185549736, "kl": 0.013786638039164245, "learning_rate": 5.795221843003413e-06, "loss": 0.0392, "num_tokens": 66703305.0, "reward": 1.0, "reward_std": 0.9870962500572205, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.3228633403778076, "sampling/importance_sampling_ratio/mean": 0.40023207664489746, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9139573574066162, "sampling/sampling_logp_difference/mean": 0.013040030375123024, "step": 627, "step_time": 79.29714062321 }, { "clip_ratio/high_max": 0.012227004743181169, "clip_ratio/high_mean": 0.0033542769524501637, "clip_ratio/low_mean": 0.0013671874912688509, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0047214644437190145, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 587.0, "completions/mean_length": 853.75, "completions/mean_terminated_length": 404.9090881347656, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.07941537257283926, "epoch": 0.3572241183162685, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02715110406279564, "kl": 0.010433306160848588, "learning_rate": 5.790102389078499e-06, "loss": -0.0398, "num_tokens": 66810895.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.6219407320022583, "sampling/importance_sampling_ratio/mean": 0.3352400064468384, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9842519760131836, "sampling/sampling_logp_difference/mean": 0.010738294571638107, "step": 628, "step_time": 72.49178026150912 }, { "clip_ratio/high_max": 0.004840411012992263, "clip_ratio/high_mean": 0.0009680821967776865, "clip_ratio/low_mean": 0.0008164686660165899, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017845508627942763, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 839.0, "completions/mean_length": 738.9500122070312, "completions/mean_terminated_length": 528.2608642578125, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.09668317809700966, "epoch": 0.35779294653014787, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.011049486696720123, "kl": 0.009343775163870305, "learning_rate": 5.784982935153584e-06, "loss": -0.0072, "num_tokens": 66911437.0, "reward": 1.0500000715255737, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.9749643802642822, "sampling/importance_sampling_ratio/mean": 0.7576127052307129, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.067681312561035, "sampling/sampling_logp_difference/mean": 0.012558377347886562, "step": 629, "step_time": 71.73812257614918 }, { "clip_ratio/high_max": 0.010005376767367125, "clip_ratio/high_mean": 0.003753855955437757, "clip_ratio/low_mean": 0.0014404296543943929, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005194285651668906, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 837.0, "completions/mean_length": 741.875, "completions/mean_terminated_length": 553.7916870117188, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.10420071380212903, "epoch": 0.3583617747440273, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03683997690677643, "kl": 0.015138130984269083, "learning_rate": 5.779863481228669e-06, "loss": -0.1001, "num_tokens": 67014272.0, "reward": 1.2000000476837158, "reward_std": 0.9922778606414795, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.721790313720703, "sampling/importance_sampling_ratio/mean": 0.4640266001224518, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.426219940185547, "sampling/sampling_logp_difference/mean": 0.013192790560424328, "step": 630, "step_time": 71.38725897367112 }, { "clip_ratio/high_max": 0.014335265499539673, "clip_ratio/high_mean": 0.005651613901136443, "clip_ratio/low_mean": 0.0023437499985448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007995363906957209, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 796.4750366210938, "completions/mean_terminated_length": 488.6470642089844, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.11467343289405107, "epoch": 0.3589306029579067, "frac_reward_zero_std": 0.0, "grad_norm": 0.025864943861961365, "kl": 0.014217214658856392, "learning_rate": 5.774744027303754e-06, "loss": 0.0296, "num_tokens": 67117027.0, "reward": 0.75, "reward_std": 0.9268086552619934, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 1.4824879169464111, "sampling/importance_sampling_ratio/mean": 0.2817211449146271, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.688368320465088, "sampling/sampling_logp_difference/mean": 0.013320005498826504, "step": 631, "step_time": 73.17441520513967 }, { "clip_ratio/high_max": 0.012662847060710192, "clip_ratio/high_mean": 0.004081180231878534, "clip_ratio/low_mean": 0.0023568794858874753, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006438059615902603, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 940.0, "completions/mean_length": 769.1749877929688, "completions/mean_terminated_length": 560.6818237304688, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.09276317805051804, "epoch": 0.3594994311717861, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026874369010329247, "kl": 0.010775981703773141, "learning_rate": 5.76962457337884e-06, "loss": 0.0166, "num_tokens": 67209434.0, "reward": 0.9000000357627869, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.2026045322418213, "sampling/importance_sampling_ratio/mean": 0.5130623579025269, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.26413631439209, "sampling/sampling_logp_difference/mean": 0.011901653371751308, "step": 632, "step_time": 68.51703669293784 }, { "clip_ratio/high_max": 0.013259812956675887, "clip_ratio/high_mean": 0.005437893109046854, "clip_ratio/low_mean": 0.002178993214329239, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007616886519826949, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 770.0, "completions/mean_length": 694.7000122070312, "completions/mean_terminated_length": 451.3043518066406, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.10063796443864703, "epoch": 0.36006825938566556, "frac_reward_zero_std": 0.0, "grad_norm": 0.03626102954149246, "kl": 0.02455132664181292, "learning_rate": 5.764505119453926e-06, "loss": -0.0553, "num_tokens": 67313742.0, "reward": 1.0750000476837158, "reward_std": 0.9710555076599121, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.6269381046295166, "sampling/importance_sampling_ratio/mean": 0.4389726519584656, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.901301145553589, "sampling/sampling_logp_difference/mean": 0.01378542184829712, "step": 633, "step_time": 71.74998009740375 }, { "clip_ratio/high_max": 0.0113574787392281, "clip_ratio/high_mean": 0.003922130155842751, "clip_ratio/low_mean": 0.001123046884458745, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005045177007559687, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 962.0, "completions/mean_length": 909.7000122070312, "completions/mean_terminated_length": 672.3077392578125, "completions/min_length": 460.0, "completions/min_terminated_length": 460.0, "entropy": 0.09090445749461651, "epoch": 0.36063708759954494, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.010594264604151249, "kl": 0.00889673357596621, "learning_rate": 5.7593856655290105e-06, "loss": 0.0019, "num_tokens": 67424474.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.9075747728347778, "sampling/importance_sampling_ratio/mean": 0.34509599208831787, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0866334438323975, "sampling/sampling_logp_difference/mean": 0.011253017000854015, "step": 634, "step_time": 73.97890181769617 }, { "clip_ratio/high_max": 0.015340470592491329, "clip_ratio/high_mean": 0.005594896530965343, "clip_ratio/low_mean": 0.002514648440410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008109545044135302, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 944.0, "completions/mean_length": 812.9249877929688, "completions/mean_terminated_length": 554.9444580078125, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.10042139142751694, "epoch": 0.36120591581342437, "frac_reward_zero_std": 0.0, "grad_norm": 0.036372750997543335, "kl": 0.016342362156137824, "learning_rate": 5.754266211604096e-06, "loss": -0.1159, "num_tokens": 67531087.0, "reward": 0.8500000238418579, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.3993189334869385, "sampling/importance_sampling_ratio/mean": 0.39627882838249207, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5523080825805664, "sampling/sampling_logp_difference/mean": 0.012762084603309631, "step": 635, "step_time": 75.9342783309985 }, { "clip_ratio/high_max": 0.008379819628316909, "clip_ratio/high_mean": 0.0019876846272381954, "clip_ratio/low_mean": 0.001992624660488218, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003980309265898541, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 835.0, "completions/mean_terminated_length": 695.3043823242188, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.11048887204378843, "epoch": 0.36177474402730375, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013971701264381409, "kl": 0.011892125476151705, "learning_rate": 5.749146757679181e-06, "loss": 0.0689, "num_tokens": 67629263.0, "reward": 0.949999988079071, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.546729326248169, "sampling/importance_sampling_ratio/mean": 0.5559467077255249, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.4028127193450928, "sampling/sampling_logp_difference/mean": 0.01204198133200407, "step": 636, "step_time": 71.63383781909943 }, { "clip_ratio/high_max": 0.008878199791070074, "clip_ratio/high_mean": 0.0028334792659734376, "clip_ratio/low_mean": 0.003892566601280123, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006726045859977603, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 853.0250244140625, "completions/mean_terminated_length": 664.0526123046875, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.11902896501123905, "epoch": 0.3623435722411832, "frac_reward_zero_std": 0.0, "grad_norm": 0.01778111793100834, "kl": 0.015952727990224957, "learning_rate": 5.744027303754266e-06, "loss": 0.002, "num_tokens": 67733352.0, "reward": 0.7750000357627869, "reward_std": 0.8911963105201721, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.1785942316055298, "sampling/importance_sampling_ratio/mean": 0.20464661717414856, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7121031284332275, "sampling/sampling_logp_difference/mean": 0.01465252973139286, "step": 637, "step_time": 72.16340819513425 }, { "clip_ratio/high_max": 0.015396225498989224, "clip_ratio/high_mean": 0.0036278100742492825, "clip_ratio/low_mean": 0.0005615234385913936, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00418933350010775, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 817.2250366210938, "completions/mean_terminated_length": 693.1599731445312, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.13495032116770744, "epoch": 0.36291240045506257, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015573223121464252, "kl": 0.01179210701957345, "learning_rate": 5.738907849829351e-06, "loss": -0.0504, "num_tokens": 67831457.0, "reward": 0.9750000238418579, "reward_std": 0.8619447350502014, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.076268434524536, "sampling/importance_sampling_ratio/mean": 0.4891051650047302, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6438465118408203, "sampling/sampling_logp_difference/mean": 0.014684932306408882, "step": 638, "step_time": 72.69291031663306 }, { "clip_ratio/high_max": 0.014024603995494545, "clip_ratio/high_mean": 0.004662733117584139, "clip_ratio/low_mean": 0.003810760681517422, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008473493857309222, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 881.9249877929688, "completions/mean_terminated_length": 724.8947143554688, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "entropy": 0.1821605134755373, "epoch": 0.363481228668942, "frac_reward_zero_std": 0.0, "grad_norm": 0.053505897521972656, "kl": 0.01545722468290478, "learning_rate": 5.733788395904437e-06, "loss": -0.1968, "num_tokens": 67940534.0, "reward": 0.8500000238418579, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.8586835861206055, "sampling/importance_sampling_ratio/mean": 0.5173639059066772, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3249902725219727, "sampling/sampling_logp_difference/mean": 0.018373744562268257, "step": 639, "step_time": 75.35517346276902 }, { "clip_ratio/high_max": 0.010657421429641545, "clip_ratio/high_mean": 0.0027139448502566665, "clip_ratio/low_mean": 0.0007889829466876108, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035029278369620442, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 884.0, "completions/mean_length": 696.7000122070312, "completions/mean_terminated_length": 520.4615478515625, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.13622082956135273, "epoch": 0.3640500568828214, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01908241957426071, "kl": 0.013598652789369226, "learning_rate": 5.728668941979522e-06, "loss": 0.0724, "num_tokens": 68036586.0, "reward": 1.274999976158142, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.3524370193481445, "sampling/importance_sampling_ratio/mean": 0.48485082387924194, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4798530340194702, "sampling/sampling_logp_difference/mean": 0.016523411497473717, "step": 640, "step_time": 71.26955753262155 }, { "clip_ratio/high_max": 0.00674262794200331, "clip_ratio/high_mean": 0.0014710746254422702, "clip_ratio/low_mean": 0.0011718750247382559, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026429496647324413, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 875.0, "completions/mean_terminated_length": 651.5, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.05191180808469653, "epoch": 0.3646188850967008, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.029683895409107208, "kl": 0.009094669891055673, "learning_rate": 5.723549488054607e-06, "loss": -0.0341, "num_tokens": 68138850.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.4070072174072266, "sampling/importance_sampling_ratio/mean": 0.4532853066921234, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2216827869415283, "sampling/sampling_logp_difference/mean": 0.008383437059819698, "step": 641, "step_time": 71.16395869199187 }, { "clip_ratio/high_max": 0.013870216731447726, "clip_ratio/high_mean": 0.0032884982356335968, "clip_ratio/low_mean": 0.0017089844041038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004997482639737427, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 883.0250244140625, "completions/mean_terminated_length": 460.1000061035156, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.10530559066683054, "epoch": 0.3651877133105802, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02507813647389412, "kl": 0.01125112408772111, "learning_rate": 5.718430034129693e-06, "loss": 0.0186, "num_tokens": 68246499.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.8987581729888916, "sampling/importance_sampling_ratio/mean": 0.42058640718460083, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2677032947540283, "sampling/sampling_logp_difference/mean": 0.012210744433104992, "step": 642, "step_time": 71.94025922939181 }, { "clip_ratio/high_max": 0.015085298684425652, "clip_ratio/high_mean": 0.0051976724789710715, "clip_ratio/low_mean": 0.0028843828185927123, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008082055137492716, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 864.5250244140625, "completions/mean_terminated_length": 669.6111450195312, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.130813954398036, "epoch": 0.36575654152445963, "frac_reward_zero_std": 0.0, "grad_norm": 0.02193291112780571, "kl": 0.01205405208747834, "learning_rate": 5.7133105802047785e-06, "loss": 0.0178, "num_tokens": 68355256.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.4572505950927734, "sampling/importance_sampling_ratio/mean": 0.3798777163028717, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.592017889022827, "sampling/sampling_logp_difference/mean": 0.014594772830605507, "step": 643, "step_time": 74.53532710904256 }, { "clip_ratio/high_max": 0.008002516435226426, "clip_ratio/high_mean": 0.0022735917664249428, "clip_ratio/low_mean": 0.0023681640450377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004641755891498178, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 797.4249877929688, "completions/mean_terminated_length": 520.5, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.12094033788889647, "epoch": 0.366325369738339, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0347021259367466, "kl": 0.012133260257542133, "learning_rate": 5.708191126279864e-06, "loss": -0.0869, "num_tokens": 68459217.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.9097108840942383, "sampling/importance_sampling_ratio/mean": 0.37405720353126526, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3567004203796387, "sampling/sampling_logp_difference/mean": 0.014730438590049744, "step": 644, "step_time": 72.50531026581302 }, { "clip_ratio/high_max": 0.010759258992038667, "clip_ratio/high_mean": 0.002913634103606455, "clip_ratio/low_mean": 0.0024414062427240424, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00535504036815837, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 918.4249877929688, "completions/mean_terminated_length": 601.7000122070312, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 0.1256016893312335, "epoch": 0.36689419795221845, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013740367256104946, "kl": 0.013398912968114018, "learning_rate": 5.7030716723549485e-06, "loss": 0.0416, "num_tokens": 68565698.0, "reward": 0.4749999940395355, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 1.818246603012085, "sampling/importance_sampling_ratio/mean": 0.4100929796695709, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.833561658859253, "sampling/sampling_logp_difference/mean": 0.014003336429595947, "step": 645, "step_time": 72.04668949195184 }, { "clip_ratio/high_max": 0.014619178604334593, "clip_ratio/high_mean": 0.005528938214411028, "clip_ratio/low_mean": 0.0026611328357830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008190070919226855, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 854.3500366210938, "completions/mean_terminated_length": 666.8421020507812, "completions/min_length": 442.0, "completions/min_terminated_length": 442.0, "entropy": 0.1421791836619377, "epoch": 0.36746302616609783, "frac_reward_zero_std": 0.0, "grad_norm": 0.03219355642795563, "kl": 0.01367167616263032, "learning_rate": 5.697952218430034e-06, "loss": 0.0271, "num_tokens": 68673032.0, "reward": 0.7250000238418579, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.9944761991500854, "sampling/importance_sampling_ratio/mean": 0.36978262662887573, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.137235403060913, "sampling/sampling_logp_difference/mean": 0.015867115929722786, "step": 646, "step_time": 74.12910611089319 }, { "clip_ratio/high_max": 0.003991185978520662, "clip_ratio/high_mean": 0.0009369334584334865, "clip_ratio/low_mean": 0.0008789062667347025, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001815839706978295, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 920.1500244140625, "completions/mean_terminated_length": 704.4615478515625, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.09149272553622723, "epoch": 0.36803185437997726, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01149013265967369, "kl": 0.008736938762012869, "learning_rate": 5.692832764505119e-06, "loss": -0.0159, "num_tokens": 68775678.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.9270483255386353, "sampling/importance_sampling_ratio/mean": 0.35658687353134155, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7193686962127686, "sampling/sampling_logp_difference/mean": 0.01103211473673582, "step": 647, "step_time": 71.64433663128875 }, { "clip_ratio/high_max": 0.002353556454181671, "clip_ratio/high_mean": 0.0004707112966571003, "clip_ratio/low_mean": 0.0006103515661379788, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010810628627950791, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 780.0, "completions/mean_length": 711.875, "completions/mean_terminated_length": 524.5999755859375, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.07806929852813482, "epoch": 0.36860068259385664, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.015204216353595257, "kl": 0.012672014650888741, "learning_rate": 5.687713310580206e-06, "loss": -0.0163, "num_tokens": 68870705.0, "reward": 1.25, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.989128589630127, "sampling/importance_sampling_ratio/mean": 0.4634905755519867, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.308777332305908, "sampling/sampling_logp_difference/mean": 0.012354983948171139, "step": 648, "step_time": 71.78441128088161 }, { "clip_ratio/high_max": 0.007187374052591622, "clip_ratio/high_mean": 0.0016780040750745684, "clip_ratio/low_mean": 0.0014892578292347025, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003167261864291504, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 968.9000244140625, "completions/mean_terminated_length": 779.1111450195312, "completions/min_length": 416.0, "completions/min_terminated_length": 416.0, "entropy": 0.06198848783969879, "epoch": 0.3691695108077361, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.006989174988120794, "kl": 0.009844392101513222, "learning_rate": 5.6825938566552894e-06, "loss": 0.0319, "num_tokens": 68975045.0, "reward": 0.45000001788139343, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.790416955947876, "sampling/importance_sampling_ratio/mean": 0.37285447120666504, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.6303954124450684, "sampling/sampling_logp_difference/mean": 0.00883163046091795, "step": 649, "step_time": 74.61212294362485 }, { "clip_ratio/high_max": 0.004394837887957692, "clip_ratio/high_mean": 0.0008789675775915384, "clip_ratio/low_mean": 0.0008544922020519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017334597687295172, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 974.0, "completions/mean_length": 885.625, "completions/mean_terminated_length": 562.75, "completions/min_length": 394.0, "completions/min_terminated_length": 394.0, "entropy": 0.06642777705565095, "epoch": 0.36973833902161546, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.012727040797472, "kl": 0.010814040899276733, "learning_rate": 5.677474402730376e-06, "loss": 0.0744, "num_tokens": 69088542.0, "reward": 0.6000000238418579, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.7815487384796143, "sampling/importance_sampling_ratio/mean": 0.36254817247390747, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1016485691070557, "sampling/sampling_logp_difference/mean": 0.0101311719045043, "step": 650, "step_time": 74.94822848518379 }, { "clip_ratio/high_max": 0.01005620474461466, "clip_ratio/high_mean": 0.0031744934676680714, "clip_ratio/low_mean": 0.0009521484462311491, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00412664188479539, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 904.625, "completions/mean_terminated_length": 626.0833740234375, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.12056935951113701, "epoch": 0.3703071672354949, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018911506980657578, "kl": 0.014718864345923066, "learning_rate": 5.672354948805461e-06, "loss": -0.0324, "num_tokens": 69198487.0, "reward": 0.5, "reward_std": 0.8164966106414795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.1011364459991455, "sampling/importance_sampling_ratio/mean": 0.41192254424095154, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.857926368713379, "sampling/sampling_logp_difference/mean": 0.013852695003151894, "step": 651, "step_time": 73.56508147530258 }, { "clip_ratio/high_max": 0.004630488692782819, "clip_ratio/high_mean": 0.0009260977567464579, "clip_ratio/low_mean": 0.0021484375174622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003074535256018862, "completions/clipped_ratio": 0.925000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 1001.9249877929688, "completions/mean_terminated_length": 729.6666870117188, "completions/min_length": 497.0, "completions/min_terminated_length": 497.0, "entropy": 0.09317577909678221, "epoch": 0.3708759954493743, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015968220308423042, "kl": 0.008740490127820522, "learning_rate": 5.667235494880547e-06, "loss": 0.0055, "num_tokens": 69308236.0, "reward": 0.15000000596046448, "reward_std": 0.5334935784339905, "rewards/format_reward/mean": 0.07500000298023224, "rewards/format_reward/std": 0.26674678921699524, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 1.9992841482162476, "sampling/importance_sampling_ratio/mean": 0.4188838005065918, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5981132984161377, "sampling/sampling_logp_difference/mean": 0.010803751647472382, "step": 652, "step_time": 76.20303216204047 }, { "clip_ratio/high_max": 0.013639109674841166, "clip_ratio/high_mean": 0.005343928663933184, "clip_ratio/low_mean": 0.0010009765774157131, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006344905181322247, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 915.0, "completions/mean_length": 742.2250366210938, "completions/mean_terminated_length": 635.3448486328125, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.10876048170030117, "epoch": 0.3714448236632537, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022678079083561897, "kl": 0.013648111023940146, "learning_rate": 5.662116040955631e-06, "loss": -0.0464, "num_tokens": 69413477.0, "reward": 1.274999976158142, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.9826133251190186, "sampling/importance_sampling_ratio/mean": 0.32413560152053833, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.863020420074463, "sampling/sampling_logp_difference/mean": 0.014208123087882996, "step": 653, "step_time": 71.83629834256135 }, { "clip_ratio/high_max": 0.016680980334058404, "clip_ratio/high_mean": 0.006828926911111921, "clip_ratio/low_mean": 0.002094058207148919, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0089229850564152, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 597.2250366210938, "completions/mean_terminated_length": 391.7407531738281, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.10130144748836756, "epoch": 0.3720136518771331, "frac_reward_zero_std": 0.0, "grad_norm": 0.039719317108392715, "kl": 0.01817485096398741, "learning_rate": 5.656996587030717e-06, "loss": -0.0944, "num_tokens": 69505854.0, "reward": 1.2000000476837158, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.7555878162384033, "sampling/importance_sampling_ratio/mean": 0.4908068776130676, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7353246212005615, "sampling/sampling_logp_difference/mean": 0.014892295002937317, "step": 654, "step_time": 70.61272160895169 }, { "clip_ratio/high_max": 0.012054216669639573, "clip_ratio/high_mean": 0.005167364390217699, "clip_ratio/low_mean": 0.0017333984578726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00690076281898655, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 942.0, "completions/mean_length": 815.6749877929688, "completions/mean_terminated_length": 607.3500366210938, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.10149439796805382, "epoch": 0.3725824800910125, "frac_reward_zero_std": 0.0, "grad_norm": 0.03861190006136894, "kl": 0.030204624985344708, "learning_rate": 5.651877133105802e-06, "loss": 0.0393, "num_tokens": 69603913.0, "reward": 0.925000011920929, "reward_std": 0.9710555076599121, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.5234601497650146, "sampling/importance_sampling_ratio/mean": 0.39768216013908386, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.867879390716553, "sampling/sampling_logp_difference/mean": 0.013483923859894276, "step": 655, "step_time": 69.72598076798022 }, { "clip_ratio/high_max": 0.015607743873260915, "clip_ratio/high_mean": 0.005788637834484689, "clip_ratio/low_mean": 0.0014463078714470612, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007234945776872337, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 796.8500366210938, "completions/mean_terminated_length": 645.4166870117188, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.1461599301546812, "epoch": 0.3731513083048919, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01913287676870823, "kl": 0.012427425244823098, "learning_rate": 5.646757679180888e-06, "loss": -0.026, "num_tokens": 69697235.0, "reward": 1.0500000715255737, "reward_std": 0.9323254823684692, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.2555181980133057, "sampling/importance_sampling_ratio/mean": 0.3843342363834381, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5912975072860718, "sampling/sampling_logp_difference/mean": 0.015788381919264793, "step": 656, "step_time": 70.79378902399912 }, { "clip_ratio/high_max": 0.010490116372238845, "clip_ratio/high_mean": 0.0029036694613751024, "clip_ratio/low_mean": 0.0018066406410071068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004710310051450506, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 990.0, "completions/mean_length": 940.2999877929688, "completions/mean_terminated_length": 766.4615478515625, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.07154289493337274, "epoch": 0.37372013651877134, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024035083130002022, "kl": 0.009072198765352368, "learning_rate": 5.641638225255973e-06, "loss": -0.0238, "num_tokens": 69806015.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.2740657329559326, "sampling/importance_sampling_ratio/mean": 0.4508272111415863, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9702582359313965, "sampling/sampling_logp_difference/mean": 0.009614313021302223, "step": 657, "step_time": 76.1104952308815 }, { "clip_ratio/high_max": 0.008998995705042034, "clip_ratio/high_mean": 0.0023065726563800126, "clip_ratio/low_mean": 0.0019100263780273963, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004216599088977091, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 870.8500366210938, "completions/mean_terminated_length": 513.5, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.09027963504195213, "epoch": 0.3742889647326507, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.012576300650835037, "kl": 0.01122873870190233, "learning_rate": 5.636518771331058e-06, "loss": -0.0097, "num_tokens": 69905673.0, "reward": 0.5, "reward_std": 0.8164966106414795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.6436406373977661, "sampling/importance_sampling_ratio/mean": 0.2179676592350006, "sampling/importance_sampling_ratio/min": 0.00182839366607368, "sampling/sampling_logp_difference/max": 2.80963397026062, "sampling/sampling_logp_difference/mean": 0.01237985398620367, "step": 658, "step_time": 74.04809681605548 }, { "clip_ratio/high_max": 0.009611172747099772, "clip_ratio/high_mean": 0.002237602930108551, "clip_ratio/low_mean": 0.0013937040821474511, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036313071177573875, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 682.6500244140625, "completions/mean_terminated_length": 536.357177734375, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.08194381650537252, "epoch": 0.37485779294653016, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02070140466094017, "kl": 0.016992511576972902, "learning_rate": 5.631399317406144e-06, "loss": -0.0757, "num_tokens": 70004619.0, "reward": 1.125, "reward_std": 0.8529736995697021, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.7123905420303345, "sampling/importance_sampling_ratio/mean": 0.4534880518913269, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.000636577606201, "sampling/sampling_logp_difference/mean": 0.012712291441857815, "step": 659, "step_time": 68.56614101794548 }, { "clip_ratio/high_max": 0.011987215839326382, "clip_ratio/high_mean": 0.003558961965609342, "clip_ratio/low_mean": 0.001269531247089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004828493139939383, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 974.0, "completions/mean_length": 726.8500366210938, "completions/mean_terminated_length": 507.2174072265625, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.11631131544709206, "epoch": 0.37542662116040953, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.009134385734796524, "kl": 0.011263521504588425, "learning_rate": 5.626279863481229e-06, "loss": -0.019, "num_tokens": 70107717.0, "reward": 0.949999988079071, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.4768338203430176, "sampling/importance_sampling_ratio/mean": 0.3863969147205353, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4822938442230225, "sampling/sampling_logp_difference/mean": 0.015358996577560902, "step": 660, "step_time": 72.45212522614747 }, { "clip_ratio/high_max": 0.014636261272244155, "clip_ratio/high_mean": 0.004699040058767423, "clip_ratio/low_mean": 0.0010009765646827873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005700016597984359, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 614.3500366210938, "completions/mean_terminated_length": 495.4193420410156, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.11918725073337555, "epoch": 0.37599544937428897, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01646733656525612, "kl": 0.015497174579650164, "learning_rate": 5.621160409556314e-06, "loss": -0.018, "num_tokens": 70211955.0, "reward": 1.4500000476837158, "reward_std": 0.8458041548728943, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.3526611328125, "sampling/importance_sampling_ratio/mean": 0.42752188444137573, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.027057409286499, "sampling/sampling_logp_difference/mean": 0.01607665978372097, "step": 661, "step_time": 70.97855401993729 }, { "clip_ratio/high_max": 0.0043162070360267535, "clip_ratio/high_mean": 0.0009620556484151166, "clip_ratio/low_mean": 0.000537109390279511, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014991650095907971, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 890.8250122070312, "completions/mean_terminated_length": 643.5, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.06820214074105024, "epoch": 0.37656427758816835, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.0092835808172822, "kl": 0.010038690001238137, "learning_rate": 5.616040955631399e-06, "loss": 0.0134, "num_tokens": 70300884.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.854801893234253, "sampling/importance_sampling_ratio/mean": 0.41511479020118713, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0517404079437256, "sampling/sampling_logp_difference/mean": 0.009642082266509533, "step": 662, "step_time": 65.98031452391297 }, { "clip_ratio/high_max": 0.012461413920391351, "clip_ratio/high_mean": 0.004627217393135652, "clip_ratio/low_mean": 0.0016185580607270822, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00624577549751848, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 932.0, "completions/mean_length": 758.625, "completions/mean_terminated_length": 599.3999633789062, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.11151347868144512, "epoch": 0.3771331058020478, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020739568397402763, "kl": 0.014430639334022999, "learning_rate": 5.610921501706485e-06, "loss": -0.0023, "num_tokens": 70413117.0, "reward": 1.2000000476837158, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.379049301147461, "sampling/importance_sampling_ratio/mean": 0.4161544740200043, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1187925338745117, "sampling/sampling_logp_difference/mean": 0.014524310827255249, "step": 663, "step_time": 74.7116752571892 }, { "clip_ratio/high_max": 0.00922765678842552, "clip_ratio/high_mean": 0.0018954315601149574, "clip_ratio/low_mean": 0.0011872522682097042, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00308268383378163, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 964.0, "completions/mean_terminated_length": 757.3333129882812, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.1329720113426447, "epoch": 0.37770193401592717, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.04357145354151726, "kl": 0.012014062493108213, "learning_rate": 5.605802047781571e-06, "loss": 0.0717, "num_tokens": 70518565.0, "reward": 0.4000000059604645, "reward_std": 0.7778998017311096, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.864943504333496, "sampling/importance_sampling_ratio/mean": 0.4289507269859314, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7612414360046387, "sampling/sampling_logp_difference/mean": 0.013972081243991852, "step": 664, "step_time": 73.44413079530932 }, { "clip_ratio/high_max": 0.010236182599328458, "clip_ratio/high_mean": 0.0023058571678120643, "clip_ratio/low_mean": 0.0022949218910071068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004600779066095129, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 962.0, "completions/mean_length": 868.8250122070312, "completions/mean_terminated_length": 580.6428833007812, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 0.12435610312968493, "epoch": 0.3782707622298066, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019148126244544983, "kl": 0.010073158831801265, "learning_rate": 5.6006825938566555e-06, "loss": 0.021, "num_tokens": 70616598.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.7476717233657837, "sampling/importance_sampling_ratio/mean": 0.3695184886455536, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4554905891418457, "sampling/sampling_logp_difference/mean": 0.014195728115737438, "step": 665, "step_time": 72.28874531015754 }, { "clip_ratio/high_max": 0.008881631307303905, "clip_ratio/high_mean": 0.0023241963936015964, "clip_ratio/low_mean": 0.0014426695233851206, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003766865935176611, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 961.0, "completions/mean_length": 713.0250244140625, "completions/mean_terminated_length": 563.2963256835938, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.08876839186996222, "epoch": 0.378839590443686, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014311195351183414, "kl": 0.014274496119469404, "learning_rate": 5.595563139931741e-06, "loss": -0.0094, "num_tokens": 70722479.0, "reward": 1.3000000715255737, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.994933009147644, "sampling/importance_sampling_ratio/mean": 0.271331250667572, "sampling/importance_sampling_ratio/min": 0.0039563532918691635, "sampling/sampling_logp_difference/max": 2.355236530303955, "sampling/sampling_logp_difference/mean": 0.013556579127907753, "step": 666, "step_time": 71.06050234520808 }, { "clip_ratio/high_max": 0.011698007059749216, "clip_ratio/high_mean": 0.004143559606745839, "clip_ratio/low_mean": 0.0021598435341729783, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006303403031779453, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 787.875, "completions/mean_terminated_length": 613.3478393554688, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.09017686080187559, "epoch": 0.3794084186575654, "frac_reward_zero_std": 0.0, "grad_norm": 0.03963512182235718, "kl": 0.013685695943422616, "learning_rate": 5.590443686006826e-06, "loss": 0.0323, "num_tokens": 70813106.0, "reward": 1.024999976158142, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.902449607849121, "sampling/importance_sampling_ratio/mean": 0.5152720212936401, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7544727325439453, "sampling/sampling_logp_difference/mean": 0.012275848537683487, "step": 667, "step_time": 71.5330403468106 }, { "clip_ratio/high_max": 0.016341551556251943, "clip_ratio/high_mean": 0.005870645254617557, "clip_ratio/low_mean": 0.001391601586874458, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00726224685786292, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 801.0, "completions/mean_length": 779.0750122070312, "completions/mean_terminated_length": 447.70587158203125, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.07816326431930065, "epoch": 0.3799772468714448, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018610095605254173, "kl": 0.01128230825997889, "learning_rate": 5.585324232081912e-06, "loss": -0.01, "num_tokens": 70918949.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.2617135047912598, "sampling/importance_sampling_ratio/mean": 0.391049325466156, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2212347984313965, "sampling/sampling_logp_difference/mean": 0.011567679233849049, "step": 668, "step_time": 73.89386581722647 }, { "clip_ratio/high_max": 0.003847347106784582, "clip_ratio/high_mean": 0.0009892496600514278, "clip_ratio/low_mean": 0.002685546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003674796491395682, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 867.7250366210938, "completions/mean_terminated_length": 503.0833435058594, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.10688007157295942, "epoch": 0.38054607508532423, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.005982804577797651, "kl": 0.008295685867778957, "learning_rate": 5.580204778156996e-06, "loss": 0.0355, "num_tokens": 71025810.0, "reward": 0.574999988079071, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.1547844409942627, "sampling/importance_sampling_ratio/mean": 0.3992560803890228, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.392796516418457, "sampling/sampling_logp_difference/mean": 0.012104067020118237, "step": 669, "step_time": 75.89669152814895 }, { "clip_ratio/high_max": 0.011721280752681196, "clip_ratio/high_mean": 0.004557158288662322, "clip_ratio/low_mean": 0.002636718752910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0071938769542612135, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 840.2250366210938, "completions/mean_terminated_length": 615.6111450195312, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.12549651972949505, "epoch": 0.38111490329920367, "frac_reward_zero_std": 0.0, "grad_norm": 0.020548885688185692, "kl": 0.013690335093997419, "learning_rate": 5.575085324232082e-06, "loss": -0.0195, "num_tokens": 71136099.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.006805419921875, "sampling/importance_sampling_ratio/mean": 0.444524347782135, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2091503143310547, "sampling/sampling_logp_difference/mean": 0.014839589595794678, "step": 670, "step_time": 74.59413945209235 }, { "clip_ratio/high_max": 0.006802506046369672, "clip_ratio/high_mean": 0.00240041411598213, "clip_ratio/low_mean": 0.0028808593669964466, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052812734502367675, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 872.0, "completions/mean_length": 942.5250244140625, "completions/mean_terminated_length": 698.1000366210938, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "entropy": 0.12041184213012457, "epoch": 0.38168373151308305, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0408688560128212, "kl": 0.01091079507023096, "learning_rate": 5.569965870307167e-06, "loss": 0.0172, "num_tokens": 71246016.0, "reward": 0.5, "reward_std": 0.8770580291748047, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.904385805130005, "sampling/importance_sampling_ratio/mean": 0.5870606899261475, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3695244789123535, "sampling/sampling_logp_difference/mean": 0.012930351309478283, "step": 671, "step_time": 75.16663165879436 }, { "clip_ratio/high_max": 0.009496069513261318, "clip_ratio/high_mean": 0.0022668641613563523, "clip_ratio/low_mean": 0.0018554687703726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004122332960832864, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 954.9750366210938, "completions/mean_terminated_length": 678.875, "completions/min_length": 419.0, "completions/min_terminated_length": 419.0, "entropy": 0.12317031156271696, "epoch": 0.3822525597269625, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.034009501338005066, "kl": 0.010226190672256052, "learning_rate": 5.564846416382252e-06, "loss": 0.018, "num_tokens": 71359687.0, "reward": 0.375, "reward_std": 0.7741827964782715, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.2049448490142822, "sampling/importance_sampling_ratio/mean": 0.3633045256137848, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8382580280303955, "sampling/sampling_logp_difference/mean": 0.012433981522917747, "step": 672, "step_time": 75.30318055790849 }, { "clip_ratio/high_max": 0.014474598574452102, "clip_ratio/high_mean": 0.004767199890920892, "clip_ratio/low_mean": 0.0036716609756695107, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008438860764726996, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 851.375, "completions/mean_terminated_length": 678.75, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.14324230700731277, "epoch": 0.38282138794084186, "frac_reward_zero_std": 0.0, "grad_norm": 0.023637767881155014, "kl": 0.01815410261042416, "learning_rate": 5.559726962457338e-06, "loss": -0.0734, "num_tokens": 71469558.0, "reward": 0.75, "reward_std": 0.8397190570831299, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.8035537004470825, "sampling/importance_sampling_ratio/mean": 0.19112318754196167, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0745108127593994, "sampling/sampling_logp_difference/mean": 0.01628062315285206, "step": 673, "step_time": 75.1428492134437 }, { "clip_ratio/high_max": 0.009765285474713892, "clip_ratio/high_mean": 0.002661030157469213, "clip_ratio/low_mean": 0.002465820312863798, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0051268505048938096, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 837.4000244140625, "completions/mean_terminated_length": 609.3333129882812, "completions/min_length": 394.0, "completions/min_terminated_length": 394.0, "entropy": 0.113966578617692, "epoch": 0.3833902161547213, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02893170528113842, "kl": 0.013215792132541537, "learning_rate": 5.554607508532424e-06, "loss": -0.0004, "num_tokens": 71576038.0, "reward": 0.824999988079071, "reward_std": 0.957761824131012, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.2768704891204834, "sampling/importance_sampling_ratio/mean": 0.3249293863773346, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9496325254440308, "sampling/sampling_logp_difference/mean": 0.014236344955861568, "step": 674, "step_time": 73.67030843137763 }, { "clip_ratio/high_max": 0.011944508994929492, "clip_ratio/high_mean": 0.003679162691696547, "clip_ratio/low_mean": 0.0020996093608118827, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005778771941550076, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 940.8250122070312, "completions/mean_terminated_length": 746.75, "completions/min_length": 502.0, "completions/min_terminated_length": 502.0, "entropy": 0.09727792255580425, "epoch": 0.3839590443686007, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02992597036063671, "kl": 0.010700311337132007, "learning_rate": 5.549488054607509e-06, "loss": -0.1103, "num_tokens": 71685487.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.336893320083618, "sampling/importance_sampling_ratio/mean": 0.43918219208717346, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7885262966156006, "sampling/sampling_logp_difference/mean": 0.012075722217559814, "step": 675, "step_time": 73.42581460252404 }, { "clip_ratio/high_max": 0.0038588536554016173, "clip_ratio/high_mean": 0.0011208108335267752, "clip_ratio/low_mean": 0.002734374997089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038551857869606465, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 980.9000244140625, "completions/mean_terminated_length": 593.0, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.13589575048536062, "epoch": 0.3845278725824801, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01201989222317934, "kl": 0.010162663005758077, "learning_rate": 5.544368600682594e-06, "loss": 0.0266, "num_tokens": 71800339.0, "reward": 0.15000000596046448, "reward_std": 0.48304590582847595, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.05000000074505806, "rewards/gpt_judge_reward/std": 0.22072142362594604, "sampling/importance_sampling_ratio/max": 1.7423462867736816, "sampling/importance_sampling_ratio/mean": 0.3120039999485016, "sampling/importance_sampling_ratio/min": 0.0004082114319317043, "sampling/sampling_logp_difference/max": 2.600823402404785, "sampling/sampling_logp_difference/mean": 0.014609942212700844, "step": 676, "step_time": 75.04069448169321 }, { "clip_ratio/high_max": 0.011712315288605168, "clip_ratio/high_mean": 0.004355876932095271, "clip_ratio/low_mean": 0.0018570692700450309, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006212946289451793, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 961.0, "completions/mean_length": 754.0, "completions/mean_terminated_length": 574.0, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 0.125865058042109, "epoch": 0.3850967007963595, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03381296247243881, "kl": 0.011931005981750786, "learning_rate": 5.539249146757679e-06, "loss": -0.036, "num_tokens": 71904683.0, "reward": 1.1750000715255737, "reward_std": 0.9841695427894592, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.394378662109375, "sampling/importance_sampling_ratio/mean": 0.4741591513156891, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7534723281860352, "sampling/sampling_logp_difference/mean": 0.014796736650168896, "step": 677, "step_time": 75.40288319415413 }, { "clip_ratio/high_max": 0.00997609255136922, "clip_ratio/high_mean": 0.002914294949732721, "clip_ratio/low_mean": 0.0021484374592546374, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005062732379883528, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 906.7000122070312, "completions/mean_terminated_length": 711.2000122070312, "completions/min_length": 432.0, "completions/min_terminated_length": 432.0, "entropy": 0.07186004286631942, "epoch": 0.3856655290102389, "frac_reward_zero_std": 0.0, "grad_norm": 0.03959112986922264, "kl": 0.008345464651938528, "learning_rate": 5.5341296928327645e-06, "loss": 0.1137, "num_tokens": 72006207.0, "reward": 0.625, "reward_std": 0.8678739070892334, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.773484945297241, "sampling/importance_sampling_ratio/mean": 0.5347962379455566, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9686377048492432, "sampling/sampling_logp_difference/mean": 0.009805726818740368, "step": 678, "step_time": 74.22628686018288 }, { "clip_ratio/high_max": 0.0074949037516489625, "clip_ratio/high_mean": 0.0017668379005044699, "clip_ratio/low_mean": 0.0022216797005967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003988517622929066, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 908.0, "completions/mean_length": 964.8250122070312, "completions/mean_terminated_length": 685.857177734375, "completions/min_length": 541.0, "completions/min_terminated_length": 541.0, "entropy": 0.10132967494428158, "epoch": 0.3862343572241183, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.008258040063083172, "kl": 0.010494415124412626, "learning_rate": 5.52901023890785e-06, "loss": 0.0244, "num_tokens": 72110824.0, "reward": 0.20000000298023224, "reward_std": 0.46409550309181213, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.5896968841552734, "sampling/importance_sampling_ratio/mean": 0.3047211766242981, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8630214929580688, "sampling/sampling_logp_difference/mean": 0.011714858002960682, "step": 679, "step_time": 73.20500824647024 }, { "clip_ratio/high_max": 0.01634732959792018, "clip_ratio/high_mean": 0.00463057411252521, "clip_ratio/low_mean": 0.002081508588162251, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006712082657031715, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 842.0, "completions/mean_length": 769.4000244140625, "completions/mean_terminated_length": 561.0909423828125, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.09604200813919306, "epoch": 0.38680318543799774, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 5383.88037109375, "kl": 847.7878007416148, "learning_rate": 5.5238907849829345e-06, "loss": 5.6862, "num_tokens": 72205440.0, "reward": 1.0750000476837158, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.9152250289916992, "sampling/importance_sampling_ratio/mean": 0.3511836528778076, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 9.498085975646973, "sampling/sampling_logp_difference/mean": 0.014050498604774475, "step": 680, "step_time": 72.37838972150348 }, { "clip_ratio/high_max": 0.00342697580344975, "clip_ratio/high_mean": 0.0006853951490484178, "clip_ratio/low_mean": 0.000610351571594947, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012957467206433648, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 940.5750122070312, "completions/mean_terminated_length": 690.2999877929688, "completions/min_length": 471.0, "completions/min_terminated_length": 471.0, "entropy": 0.1513774460181594, "epoch": 0.3873720136518771, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.01025315560400486, "kl": 0.009583567385561764, "learning_rate": 5.518771331058021e-06, "loss": 0.0248, "num_tokens": 72308551.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.909080982208252, "sampling/importance_sampling_ratio/mean": 0.5026288032531738, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8761441707611084, "sampling/sampling_logp_difference/mean": 0.014759495854377747, "step": 681, "step_time": 72.31783312489279 }, { "clip_ratio/high_max": 0.010637229832354933, "clip_ratio/high_mean": 0.0027659879997372627, "clip_ratio/low_mean": 0.0023437499712599674, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005109737976454198, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 904.9500122070312, "completions/mean_terminated_length": 591.0909423828125, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.1093313293531537, "epoch": 0.38794084186575656, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0150835569947958, "kl": 0.009487292729318142, "learning_rate": 5.513651877133106e-06, "loss": 0.0527, "num_tokens": 72420149.0, "reward": 0.4749999940395355, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.745816230773926, "sampling/importance_sampling_ratio/mean": 0.36466363072395325, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0742835998535156, "sampling/sampling_logp_difference/mean": 0.012500191107392311, "step": 682, "step_time": 75.16133780172095 }, { "clip_ratio/high_max": 0.012801983626559377, "clip_ratio/high_mean": 0.006025005714036524, "clip_ratio/low_mean": 0.001871591270173667, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007896596973296255, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 850.0, "completions/mean_length": 690.9500122070312, "completions/mean_terminated_length": 511.61541748046875, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.10400373674929142, "epoch": 0.38850967007963594, "frac_reward_zero_std": 0.0, "grad_norm": 0.044563740491867065, "kl": 0.012674338417127728, "learning_rate": 5.508532423208192e-06, "loss": -0.1504, "num_tokens": 72522731.0, "reward": 1.225000023841858, "reward_std": 0.9736924767494202, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.84434175491333, "sampling/importance_sampling_ratio/mean": 0.5514742732048035, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.837871789932251, "sampling/sampling_logp_difference/mean": 0.013729690574109554, "step": 683, "step_time": 74.94155117846094 }, { "clip_ratio/high_max": 0.015985682257451117, "clip_ratio/high_mean": 0.00553707976359874, "clip_ratio/low_mean": 0.0012451171878637979, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006782196898711845, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 860.0, "completions/mean_length": 706.375, "completions/mean_terminated_length": 553.4444580078125, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.11840766854584217, "epoch": 0.3890784982935154, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.043060969561338425, "kl": 0.01605272968299687, "learning_rate": 5.503412969283276e-06, "loss": -0.1351, "num_tokens": 72623138.0, "reward": 1.225000023841858, "reward_std": 0.9195177555084229, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.5046558380126953, "sampling/importance_sampling_ratio/mean": 0.5904008150100708, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.626236915588379, "sampling/sampling_logp_difference/mean": 0.015260428190231323, "step": 684, "step_time": 72.78532677795738 }, { "clip_ratio/high_max": 0.005748901545302942, "clip_ratio/high_mean": 0.0011497803061502054, "clip_ratio/low_mean": 0.0022705078299622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003420288019697182, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 945.0, "completions/mean_length": 874.9000244140625, "completions/mean_terminated_length": 527.0, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.10840188525617123, "epoch": 0.38964732650739475, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.008372887969017029, "kl": 0.011795259604696184, "learning_rate": 5.498293515358362e-06, "loss": 0.0659, "num_tokens": 72735190.0, "reward": 0.574999988079071, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.822488784790039, "sampling/importance_sampling_ratio/mean": 0.3879915773868561, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2439141273498535, "sampling/sampling_logp_difference/mean": 0.013394785113632679, "step": 685, "step_time": 73.43578089866787 }, { "clip_ratio/high_max": 0.014948231284506619, "clip_ratio/high_mean": 0.005098318797536194, "clip_ratio/low_mean": 0.001733398465148639, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00683171721175313, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 861.3250122070312, "completions/mean_terminated_length": 662.5, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.17023733258247375, "epoch": 0.3902161547212742, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.05946945771574974, "kl": 0.01373460772447288, "learning_rate": 5.493174061433447e-06, "loss": -0.0547, "num_tokens": 72850619.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.97900390625, "sampling/importance_sampling_ratio/mean": 0.3693126142024994, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.980207920074463, "sampling/sampling_logp_difference/mean": 0.016823485493659973, "step": 686, "step_time": 78.69929177430458 }, { "clip_ratio/high_max": 0.011358034040313214, "clip_ratio/high_mean": 0.003810891052125953, "clip_ratio/low_mean": 0.0015624999978172127, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005373391148168594, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 835.4249877929688, "completions/mean_terminated_length": 627.0, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "entropy": 0.08625953923910856, "epoch": 0.39078498293515357, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024080460891127586, "kl": 0.021419846802018583, "learning_rate": 5.4880546075085325e-06, "loss": 0.0344, "num_tokens": 72958348.0, "reward": 0.9000000357627869, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.891356945037842, "sampling/importance_sampling_ratio/mean": 0.39985281229019165, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5902037620544434, "sampling/sampling_logp_difference/mean": 0.011904686689376831, "step": 687, "step_time": 73.40164712583646 }, { "clip_ratio/high_max": 0.01224871992599219, "clip_ratio/high_mean": 0.004739591502584517, "clip_ratio/low_mean": 0.0017895110504468903, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006529102451168001, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 746.2250366210938, "completions/mean_terminated_length": 653.6333618164062, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "entropy": 0.10931319929659367, "epoch": 0.391353811149033, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03224310278892517, "kl": 0.011000219325069338, "learning_rate": 5.482935153583617e-06, "loss": 0.133, "num_tokens": 73059845.0, "reward": 1.399999976158142, "reward_std": 0.8711913824081421, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.4100637435913086, "sampling/importance_sampling_ratio/mean": 0.46350517868995667, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.604811191558838, "sampling/sampling_logp_difference/mean": 0.014008372090756893, "step": 688, "step_time": 72.01827693427913 }, { "clip_ratio/high_max": 0.008349072188138962, "clip_ratio/high_mean": 0.001859495765529573, "clip_ratio/low_mean": 0.0020751953561557457, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003934691194444895, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 792.0, "completions/mean_length": 957.3250122070312, "completions/mean_terminated_length": 579.5, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.11439959332346916, "epoch": 0.3919226393629124, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.05178987979888916, "kl": 0.008312734018545598, "learning_rate": 5.477815699658703e-06, "loss": -0.1366, "num_tokens": 73168730.0, "reward": 0.2750000059604645, "reward_std": 0.6788942813873291, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.9554107189178467, "sampling/importance_sampling_ratio/mean": 0.59920334815979, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.189944267272949, "sampling/sampling_logp_difference/mean": 0.01219781395047903, "step": 689, "step_time": 73.5483097883407 }, { "clip_ratio/high_max": 0.015698396833613515, "clip_ratio/high_mean": 0.004078624639078043, "clip_ratio/low_mean": 0.0005371093830035534, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004615734040271491, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 896.0, "completions/mean_length": 841.25, "completions/mean_terminated_length": 658.5, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.1634560152888298, "epoch": 0.3924914675767918, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.062211886048316956, "kl": 0.022173094796016812, "learning_rate": 5.472696245733789e-06, "loss": -0.0066, "num_tokens": 73271692.0, "reward": 0.800000011920929, "reward_std": 0.8828856945037842, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.5772836208343506, "sampling/importance_sampling_ratio/mean": 0.3020961880683899, "sampling/importance_sampling_ratio/min": 0.0057592387311160564, "sampling/sampling_logp_difference/max": 2.6760356426239014, "sampling/sampling_logp_difference/mean": 0.017540208995342255, "step": 690, "step_time": 76.76035547605716 }, { "clip_ratio/high_max": 0.002870622876798734, "clip_ratio/high_mean": 0.0005741245768149383, "clip_ratio/low_mean": 0.001538085958600277, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021122105390531942, "completions/clipped_ratio": 0.925000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 996.0750122070312, "completions/mean_terminated_length": 651.6666870117188, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.05209926189854741, "epoch": 0.3930602957906712, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015108269639313221, "kl": 0.007824167958460748, "learning_rate": 5.467576791808874e-06, "loss": -0.0541, "num_tokens": 73366255.0, "reward": 0.15000000596046448, "reward_std": 0.5334935784339905, "rewards/format_reward/mean": 0.07500000298023224, "rewards/format_reward/std": 0.26674678921699524, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.4638478755950928, "sampling/importance_sampling_ratio/mean": 0.4390665590763092, "sampling/importance_sampling_ratio/min": 0.01379796490073204, "sampling/sampling_logp_difference/max": 2.117732286453247, "sampling/sampling_logp_difference/mean": 0.0071419659070670605, "step": 691, "step_time": 70.51140640978701 }, { "clip_ratio/high_max": 0.01968670974019915, "clip_ratio/high_mean": 0.0057473923079669476, "clip_ratio/low_mean": 0.0024955069238785654, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008242899202741683, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 926.0, "completions/mean_length": 804.3250122070312, "completions/mean_terminated_length": 561.5263061523438, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.13287709653377533, "epoch": 0.39362912400455063, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014605699107050896, "kl": 0.01285451150033623, "learning_rate": 5.462457337883959e-06, "loss": 0.0069, "num_tokens": 73474164.0, "reward": 0.800000011920929, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.9295984506607056, "sampling/importance_sampling_ratio/mean": 0.28532373905181885, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0267186164855957, "sampling/sampling_logp_difference/mean": 0.015185623429715633, "step": 692, "step_time": 73.70861993171275 }, { "clip_ratio/high_max": 0.010679998667910695, "clip_ratio/high_mean": 0.00307184478151612, "clip_ratio/low_mean": 0.001074218771464075, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004146063554799184, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 803.3500366210938, "completions/mean_terminated_length": 533.6666870117188, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.11171738337725401, "epoch": 0.39419795221843, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025136422365903854, "kl": 0.009266567300073802, "learning_rate": 5.457337883959044e-06, "loss": 0.1405, "num_tokens": 73573634.0, "reward": 0.9000000357627869, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.1960673332214355, "sampling/importance_sampling_ratio/mean": 0.501594066619873, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8630226850509644, "sampling/sampling_logp_difference/mean": 0.01211504079401493, "step": 693, "step_time": 71.78215812775306 }, { "clip_ratio/high_max": 0.012423074571415782, "clip_ratio/high_mean": 0.0037459973245859146, "clip_ratio/low_mean": 0.0024925523794081528, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062385496275965124, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 787.4500122070312, "completions/mean_terminated_length": 612.6087036132812, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.11625118367373943, "epoch": 0.39476678043230945, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013798240572214127, "kl": 0.012467247783206403, "learning_rate": 5.45221843003413e-06, "loss": 0.0723, "num_tokens": 73676324.0, "reward": 0.949999988079071, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.3473012447357178, "sampling/importance_sampling_ratio/mean": 0.5492976903915405, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9466423988342285, "sampling/sampling_logp_difference/mean": 0.014134091325104237, "step": 694, "step_time": 72.62691012001596 }, { "clip_ratio/high_max": 0.013384836143814027, "clip_ratio/high_mean": 0.005487425427418202, "clip_ratio/low_mean": 0.003076171880820766, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.008563597220927477, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 828.5499877929688, "completions/mean_terminated_length": 612.5263061523438, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.13554854225367308, "epoch": 0.39533560864618883, "frac_reward_zero_std": 0.0, "grad_norm": 0.026701495051383972, "kl": 0.010427388304378837, "learning_rate": 5.447098976109216e-06, "loss": -0.0416, "num_tokens": 73782778.0, "reward": 0.949999988079071, "reward_std": 1.0114725828170776, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.577860951423645, "sampling/importance_sampling_ratio/mean": 0.30390214920043945, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3269858360290527, "sampling/sampling_logp_difference/mean": 0.014652076177299023, "step": 695, "step_time": 72.74517998914234 }, { "clip_ratio/high_max": 0.008157166535966098, "clip_ratio/high_mean": 0.002506558405002579, "clip_ratio/low_mean": 0.002148437466530595, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004654995835153386, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 896.0, "completions/mean_length": 946.9249877929688, "completions/mean_terminated_length": 681.4444580078125, "completions/min_length": 488.0, "completions/min_terminated_length": 488.0, "entropy": 0.07543343910947442, "epoch": 0.39590443686006827, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04523039609193802, "kl": 0.011925551516469568, "learning_rate": 5.4419795221843e-06, "loss": -0.077, "num_tokens": 73894455.0, "reward": 0.45000001788139343, "reward_std": 0.8458041548728943, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290207743644714, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.9273018836975098, "sampling/importance_sampling_ratio/mean": 0.47464603185653687, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.636140823364258, "sampling/sampling_logp_difference/mean": 0.010121941566467285, "step": 696, "step_time": 74.81978923385032 }, { "clip_ratio/high_max": 0.013577300938777626, "clip_ratio/high_mean": 0.004810618585906923, "clip_ratio/low_mean": 0.002573442558059469, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007384061347693205, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 891.0, "completions/mean_length": 767.5, "completions/mean_terminated_length": 577.9130249023438, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.10887498501688242, "epoch": 0.39647326507394764, "frac_reward_zero_std": 0.0, "grad_norm": 0.026658665388822556, "kl": 0.015141587238758802, "learning_rate": 5.436860068259386e-06, "loss": 0.1111, "num_tokens": 73986059.0, "reward": 1.0, "reward_std": 0.9336995482444763, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.3419666290283203, "sampling/importance_sampling_ratio/mean": 0.4299888014793396, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8915624618530273, "sampling/sampling_logp_difference/mean": 0.012999370694160461, "step": 697, "step_time": 69.46348359296098 }, { "clip_ratio/high_max": 0.01337512896861881, "clip_ratio/high_mean": 0.005255280208075419, "clip_ratio/low_mean": 0.0013671874985448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006622467655688524, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 842.0, "completions/mean_length": 661.7999877929688, "completions/mean_terminated_length": 524.413818359375, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.12183931469917297, "epoch": 0.3970420932878271, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.025098741054534912, "kl": 0.018686364288441837, "learning_rate": 5.4317406143344715e-06, "loss": 0.0242, "num_tokens": 74090675.0, "reward": 1.100000023841858, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.5408036708831787, "sampling/importance_sampling_ratio/mean": 0.49893084168434143, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.761935234069824, "sampling/sampling_logp_difference/mean": 0.015079902485013008, "step": 698, "step_time": 73.1883386420086 }, { "clip_ratio/high_max": 0.00979417993221432, "clip_ratio/high_mean": 0.002895425190217793, "clip_ratio/low_mean": 0.001855468755820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004750894062453881, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 785.0499877929688, "completions/mean_terminated_length": 608.434814453125, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.11051560007035732, "epoch": 0.39761092150170646, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03189362213015556, "kl": 0.011584254913032055, "learning_rate": 5.426621160409557e-06, "loss": -0.0423, "num_tokens": 74201205.0, "reward": 1.149999976158142, "reward_std": 1.0012813806533813, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.8842092752456665, "sampling/importance_sampling_ratio/mean": 0.5066415667533875, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4939985275268555, "sampling/sampling_logp_difference/mean": 0.013748315162956715, "step": 699, "step_time": 73.41291689290665 }, { "clip_ratio/high_max": 0.00830822781426832, "clip_ratio/high_mean": 0.001995473459828645, "clip_ratio/low_mean": 0.0014999650193203706, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034954384609591216, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 866.0, "completions/mean_length": 931.4500122070312, "completions/mean_terminated_length": 653.7999877929688, "completions/min_length": 469.0, "completions/min_terminated_length": 469.0, "entropy": 0.14178757835179567, "epoch": 0.3981797497155859, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011962663382291794, "kl": 0.010018853761721402, "learning_rate": 5.4215017064846415e-06, "loss": 0.0274, "num_tokens": 74306079.0, "reward": 0.4000000059604645, "reward_std": 0.7442084550857544, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 1.309945821762085, "sampling/importance_sampling_ratio/mean": 0.3127623498439789, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8883635997772217, "sampling/sampling_logp_difference/mean": 0.014760568737983704, "step": 700, "step_time": 71.6238708619494 }, { "clip_ratio/high_max": 0.009884976549074054, "clip_ratio/high_mean": 0.0033463339641457424, "clip_ratio/low_mean": 0.0031524298901786096, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006498763803392649, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 975.0, "completions/mean_length": 828.4000244140625, "completions/mean_terminated_length": 563.7647094726562, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.13632158003747463, "epoch": 0.3987485779294653, "frac_reward_zero_std": 0.0, "grad_norm": 0.01931491121649742, "kl": 0.01196313404943794, "learning_rate": 5.416382252559727e-06, "loss": -0.0366, "num_tokens": 74416727.0, "reward": 0.7750000357627869, "reward_std": 0.9469925761222839, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.873379111289978, "sampling/importance_sampling_ratio/mean": 0.2786586880683899, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.433176040649414, "sampling/sampling_logp_difference/mean": 0.01544251199811697, "step": 701, "step_time": 73.60266550583765 }, { "clip_ratio/high_max": 0.010018904227763414, "clip_ratio/high_mean": 0.002684238672372885, "clip_ratio/low_mean": 0.0012939453044964466, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003978183929575607, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 803.0, "completions/mean_length": 867.7000122070312, "completions/mean_terminated_length": 503.0, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.07102138735353947, "epoch": 0.3993174061433447, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0217647235840559, "kl": 0.009203987952787429, "learning_rate": 5.411262798634812e-06, "loss": 0.0074, "num_tokens": 74527419.0, "reward": 0.6000000238418579, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.235952615737915, "sampling/importance_sampling_ratio/mean": 0.37991032004356384, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0358994007110596, "sampling/sampling_logp_difference/mean": 0.00877335574477911, "step": 702, "step_time": 75.62200646894053 }, { "clip_ratio/high_max": 0.009352424938697368, "clip_ratio/high_mean": 0.003132080048089847, "clip_ratio/low_mean": 0.0017822265508584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0049143066426040605, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 789.6749877929688, "completions/mean_terminated_length": 597.95458984375, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.08347438275814056, "epoch": 0.3998862343572241, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030892467126250267, "kl": 0.009968666592612863, "learning_rate": 5.406143344709899e-06, "loss": 0.0362, "num_tokens": 74635190.0, "reward": 1.0500000715255737, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.6954922676086426, "sampling/importance_sampling_ratio/mean": 0.49906986951828003, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.310429811477661, "sampling/sampling_logp_difference/mean": 0.011631096713244915, "step": 703, "step_time": 73.42706594546326 }, { "clip_ratio/high_max": 0.013343715632800013, "clip_ratio/high_mean": 0.0040811324724927545, "clip_ratio/low_mean": 0.0034649735025595874, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0075461058877408504, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 841.4500122070312, "completions/mean_terminated_length": 618.3333129882812, "completions/min_length": 436.0, "completions/min_terminated_length": 436.0, "entropy": 0.13983841240406036, "epoch": 0.4004550625711035, "frac_reward_zero_std": 0.0, "grad_norm": 0.04503825306892395, "kl": 0.03384679969167337, "learning_rate": 5.401023890784983e-06, "loss": 0.0232, "num_tokens": 74739632.0, "reward": 0.7750000357627869, "reward_std": 0.9195177555084229, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.6896374225616455, "sampling/importance_sampling_ratio/mean": 0.3885204493999481, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9778478145599365, "sampling/sampling_logp_difference/mean": 0.01562996581196785, "step": 704, "step_time": 72.2239140283782 }, { "clip_ratio/high_max": 0.010465021594427526, "clip_ratio/high_mean": 0.004066719411639497, "clip_ratio/low_mean": 0.002221679715148639, "clip_ratio/low_min": 0.0006103515625, "clip_ratio/region_mean": 0.006288399104960263, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 962.0, "completions/mean_length": 844.0750122070312, "completions/mean_terminated_length": 544.2000122070312, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.07015818543732166, "epoch": 0.40102389078498296, "frac_reward_zero_std": 0.0, "grad_norm": 0.024523809552192688, "kl": 0.0091199716553092, "learning_rate": 5.395904436860069e-06, "loss": 0.0235, "num_tokens": 74841499.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.5403833389282227, "sampling/importance_sampling_ratio/mean": 0.4624256193637848, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6750967502593994, "sampling/sampling_logp_difference/mean": 0.009875264950096607, "step": 705, "step_time": 73.31856629974209 }, { "clip_ratio/high_max": 0.012988802860490978, "clip_ratio/high_mean": 0.004115253628697246, "clip_ratio/low_mean": 0.0022460937398136593, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006361347303027287, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 831.0, "completions/mean_terminated_length": 638.0, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.12143568322062492, "epoch": 0.40159271899886234, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020544007420539856, "kl": 0.011001508159097284, "learning_rate": 5.390784982935154e-06, "loss": -0.0026, "num_tokens": 74949819.0, "reward": 0.949999988079071, "reward_std": 0.9857965111732483, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.0621016025543213, "sampling/importance_sampling_ratio/mean": 0.35818764567375183, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6133965253829956, "sampling/sampling_logp_difference/mean": 0.014563190750777721, "step": 706, "step_time": 73.04664713377133 }, { "clip_ratio/high_max": 0.014183043909724802, "clip_ratio/high_mean": 0.004332730721216649, "clip_ratio/low_mean": 0.002685546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007018277538008988, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 908.7999877929688, "completions/mean_terminated_length": 640.0, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.10008195135742426, "epoch": 0.4021615472127418, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015652241185307503, "kl": 0.009874342998955399, "learning_rate": 5.385665529010239e-06, "loss": 0.0586, "num_tokens": 75061643.0, "reward": 0.42500001192092896, "reward_std": 0.7120753526687622, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 1.4281572103500366, "sampling/importance_sampling_ratio/mean": 0.3455846905708313, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.074666976928711, "sampling/sampling_logp_difference/mean": 0.011711920611560345, "step": 707, "step_time": 73.48972555878572 }, { "clip_ratio/high_max": 0.005823532410431653, "clip_ratio/high_mean": 0.001466951274778694, "clip_ratio/low_mean": 0.0012695312507275958, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002736482536420226, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 683.0250244140625, "completions/mean_terminated_length": 569.36669921875, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.15513426810503006, "epoch": 0.40273037542662116, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.09827281534671783, "kl": 0.019486833130940795, "learning_rate": 5.380546075085324e-06, "loss": -0.0331, "num_tokens": 75168404.0, "reward": 1.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.75, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.0530576705932617, "sampling/importance_sampling_ratio/mean": 0.38556939363479614, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.388731002807617, "sampling/sampling_logp_difference/mean": 0.01700570620596409, "step": 708, "step_time": 131.70713936304674 }, { "clip_ratio/high_max": 0.004555358784273267, "clip_ratio/high_mean": 0.0009946838108589873, "clip_ratio/low_mean": 0.0024658203074068297, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034605040564201772, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 864.1500244140625, "completions/mean_terminated_length": 719.5238037109375, "completions/min_length": 438.0, "completions/min_terminated_length": 438.0, "entropy": 0.14795117918401957, "epoch": 0.4032992036405006, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03508535400032997, "kl": 0.012422330910339952, "learning_rate": 5.3754266211604095e-06, "loss": -0.0222, "num_tokens": 75270362.0, "reward": 1.024999976158142, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.6486854553222656, "sampling/importance_sampling_ratio/mean": 0.41236430406570435, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8062936067581177, "sampling/sampling_logp_difference/mean": 0.015688098967075348, "step": 709, "step_time": 71.74108717706986 }, { "clip_ratio/high_max": 0.00772446405608207, "clip_ratio/high_mean": 0.0015448927879333496, "clip_ratio/low_mean": 0.00029296875618456397, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018378615477558924, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 661.9000244140625, "completions/mean_terminated_length": 487.5555725097656, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.13788017071783543, "epoch": 0.40386803185437997, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.022989466786384583, "kl": 0.015342194819822907, "learning_rate": 5.370307167235495e-06, "loss": 0.0313, "num_tokens": 75372174.0, "reward": 1.2000000476837158, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.6518579721450806, "sampling/importance_sampling_ratio/mean": 0.39028751850128174, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1977813243865967, "sampling/sampling_logp_difference/mean": 0.016702227294445038, "step": 710, "step_time": 69.13870630110614 }, { "clip_ratio/high_max": 0.010529255610890687, "clip_ratio/high_mean": 0.0023121217236621305, "clip_ratio/low_mean": 0.0006835937711002771, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029957155347801745, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 950.0, "completions/mean_length": 938.625, "completions/mean_terminated_length": 644.5555419921875, "completions/min_length": 452.0, "completions/min_terminated_length": 452.0, "entropy": 0.12136669270694256, "epoch": 0.4044368600682594, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.0060135372914373875, "kl": 0.009095622110180557, "learning_rate": 5.3651877133105796e-06, "loss": 0.0255, "num_tokens": 75477023.0, "reward": 0.4000000059604645, "reward_std": 0.8101914525032043, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.5822159051895142, "sampling/importance_sampling_ratio/mean": 0.3370535969734192, "sampling/importance_sampling_ratio/min": 0.005063303746283054, "sampling/sampling_logp_difference/max": 2.0744102001190186, "sampling/sampling_logp_difference/mean": 0.012686103582382202, "step": 711, "step_time": 73.27056763251312 }, { "clip_ratio/high_max": 0.008104630978778005, "clip_ratio/high_mean": 0.0016209261666517705, "clip_ratio/low_mean": 0.0013196526851970702, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029405788518488407, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 831.7000122070312, "completions/mean_terminated_length": 511.20001220703125, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.13093914929777384, "epoch": 0.4050056882821388, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.00531746307387948, "kl": 0.010413558804430068, "learning_rate": 5.360068259385666e-06, "loss": -0.0, "num_tokens": 75581587.0, "reward": 0.675000011920929, "reward_std": 0.9167249798774719, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.8293254375457764, "sampling/importance_sampling_ratio/mean": 0.3433915972709656, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6570611000061035, "sampling/sampling_logp_difference/mean": 0.014919864945113659, "step": 712, "step_time": 71.9762823516503 }, { "clip_ratio/high_max": 0.01323223882354796, "clip_ratio/high_mean": 0.006216036912519485, "clip_ratio/low_mean": 0.001475074008340016, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00769111089175567, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 752.9500122070312, "completions/mean_terminated_length": 607.0, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.10782002937048674, "epoch": 0.4055745164960182, "frac_reward_zero_std": 0.0, "grad_norm": 0.03987874463200569, "kl": 0.012966787442564964, "learning_rate": 5.354948805460751e-06, "loss": -0.0592, "num_tokens": 75681113.0, "reward": 1.125, "reward_std": 0.9111136794090271, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.8202075958251953, "sampling/importance_sampling_ratio/mean": 0.578809916973114, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.960340142250061, "sampling/sampling_logp_difference/mean": 0.013020305894315243, "step": 713, "step_time": 72.11323549388908 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 656.0, "completions/mean_length": 821.25, "completions/mean_terminated_length": 517.125, "completions/min_length": 392.0, "completions/min_terminated_length": 392.0, "entropy": 0.06228884356096387, "epoch": 0.4061433447098976, "frac_reward_zero_std": 1.0, "grad_norm": 0.0014861662639304996, "kl": 0.008701022830791771, "learning_rate": 5.349829351535837e-06, "loss": 0.0001, "num_tokens": 75781163.0, "reward": 0.800000011920929, "reward_std": 0.9922778606414795, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.7988483905792236, "sampling/importance_sampling_ratio/mean": 0.5131875872612, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.8522729873657227, "sampling/sampling_logp_difference/mean": 0.009681894443929195, "step": 714, "step_time": 70.39767898456194 }, { "clip_ratio/high_max": 0.011514788842760026, "clip_ratio/high_mean": 0.0038671735674142838, "clip_ratio/low_mean": 0.0031532745088043157, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.007020448159892112, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 871.1749877929688, "completions/mean_terminated_length": 718.3500366210938, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.12875920813530684, "epoch": 0.40671217292377704, "frac_reward_zero_std": 0.0, "grad_norm": 0.029226405546069145, "kl": 0.012852646643295884, "learning_rate": 5.344709897610921e-06, "loss": -0.1083, "num_tokens": 75878010.0, "reward": 0.7750000357627869, "reward_std": 0.8619447350502014, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 1.9567363262176514, "sampling/importance_sampling_ratio/mean": 0.4056420922279358, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.147136926651001, "sampling/sampling_logp_difference/mean": 0.013714957050979137, "step": 715, "step_time": 70.48108163941652 }, { "clip_ratio/high_max": 0.014379491214640439, "clip_ratio/high_mean": 0.0067337542423047125, "clip_ratio/low_mean": 0.0017226040945388377, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008456358395051211, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 774.7000122070312, "completions/mean_terminated_length": 625.1199951171875, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.13510635774582624, "epoch": 0.4072810011376564, "frac_reward_zero_std": 0.0, "grad_norm": 0.02257925271987915, "kl": 0.017630436224862933, "learning_rate": 5.339590443686007e-06, "loss": -0.0467, "num_tokens": 75975046.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 1.6091524362564087, "sampling/importance_sampling_ratio/mean": 0.35896268486976624, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.284545421600342, "sampling/sampling_logp_difference/mean": 0.014982110820710659, "step": 716, "step_time": 70.38849227968603 }, { "clip_ratio/high_max": 0.0048077239480335265, "clip_ratio/high_mean": 0.0010446933592902496, "clip_ratio/low_mean": 0.0006103515770519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016550449072383344, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 964.0499877929688, "completions/mean_terminated_length": 724.25, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.09592437790706754, "epoch": 0.40784982935153585, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.02185400016605854, "kl": 0.01363214838784188, "learning_rate": 5.334470989761092e-06, "loss": 0.0494, "num_tokens": 76076976.0, "reward": 0.375, "reward_std": 0.7741827964782715, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.961172342300415, "sampling/importance_sampling_ratio/mean": 0.4169476628303528, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.6934337615966797, "sampling/sampling_logp_difference/mean": 0.010792438872158527, "step": 717, "step_time": 72.77597741410136 }, { "clip_ratio/high_max": 0.008070701267570257, "clip_ratio/high_mean": 0.0031242307304637507, "clip_ratio/low_mean": 0.002856445382349193, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.005980676040053368, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 942.0, "completions/mean_length": 896.0750122070312, "completions/mean_terminated_length": 704.1875, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.11840574536472559, "epoch": 0.40841865756541523, "frac_reward_zero_std": 0.0, "grad_norm": 0.01718536578118801, "kl": 0.013268622569739819, "learning_rate": 5.329351535836178e-06, "loss": 0.0381, "num_tokens": 76187075.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.88519024848938, "sampling/importance_sampling_ratio/mean": 0.4596827030181885, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9412100315093994, "sampling/sampling_logp_difference/mean": 0.013267998583614826, "step": 718, "step_time": 73.74376191175543 }, { "clip_ratio/high_max": 0.0046013344544917345, "clip_ratio/high_mean": 0.0009202668734360486, "clip_ratio/low_mean": 0.0007324218786379788, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016526887520740274, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 875.1749877929688, "completions/mean_terminated_length": 482.8182067871094, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.156476566568017, "epoch": 0.40898748577929467, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.010472171939909458, "kl": 0.009603685815818608, "learning_rate": 5.324232081911262e-06, "loss": -0.001, "num_tokens": 76291074.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.91957426071167, "sampling/importance_sampling_ratio/mean": 0.3788473606109619, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6526494026184082, "sampling/sampling_logp_difference/mean": 0.016647038981318474, "step": 719, "step_time": 71.21231123316102 }, { "clip_ratio/high_max": 0.016010379418730736, "clip_ratio/high_mean": 0.004703862126916647, "clip_ratio/low_mean": 0.0026367187092546374, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007340580807067454, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 932.5750122070312, "completions/mean_terminated_length": 795.4375, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "entropy": 0.11622047051787376, "epoch": 0.40955631399317405, "frac_reward_zero_std": 0.0, "grad_norm": 0.01632346771657467, "kl": 0.009071864595171064, "learning_rate": 5.3191126279863485e-06, "loss": 0.0382, "num_tokens": 76394361.0, "reward": 0.75, "reward_std": 0.9540736079216003, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.1561580896377563, "sampling/importance_sampling_ratio/mean": 0.2855834364891052, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3919930458068848, "sampling/sampling_logp_difference/mean": 0.012086833827197552, "step": 720, "step_time": 71.88039991795085 }, { "clip_ratio/high_max": 0.014677136205136776, "clip_ratio/high_mean": 0.004131449313717894, "clip_ratio/low_mean": 0.0013183593764551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005449808755656704, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 904.875, "completions/mean_terminated_length": 657.4615478515625, "completions/min_length": 443.0, "completions/min_terminated_length": 443.0, "entropy": 0.11778833530843258, "epoch": 0.4101251422070535, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.023257136344909668, "kl": 0.009662545402534306, "learning_rate": 5.313993174061434e-06, "loss": -0.0365, "num_tokens": 76500964.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.077077627182007, "sampling/importance_sampling_ratio/mean": 0.3063604533672333, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2351388931274414, "sampling/sampling_logp_difference/mean": 0.013149715960025787, "step": 721, "step_time": 74.10003984719515 }, { "clip_ratio/high_max": 0.014402416709344834, "clip_ratio/high_mean": 0.00440705668006558, "clip_ratio/low_mean": 0.0019286057104181964, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006335662474157289, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 691.5250244140625, "completions/mean_terminated_length": 531.4444580078125, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.12125781644135714, "epoch": 0.41069397042093286, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04459884390234947, "kl": 0.012800263823010027, "learning_rate": 5.308873720136519e-06, "loss": -0.049, "num_tokens": 76603113.0, "reward": 1.225000023841858, "reward_std": 0.9195176959037781, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.822873830795288, "sampling/importance_sampling_ratio/mean": 0.3961341977119446, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.946645975112915, "sampling/sampling_logp_difference/mean": 0.015630731359124184, "step": 722, "step_time": 77.93775821407326 }, { "clip_ratio/high_max": 0.0014085943112149835, "clip_ratio/high_mean": 0.0002817188578774221, "clip_ratio/low_mean": 0.001171875021100277, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014535938717017416, "completions/clipped_ratio": 0.949999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 1021.3250122070312, "completions/mean_terminated_length": 970.5, "completions/min_length": 948.0, "completions/min_terminated_length": 948.0, "entropy": 0.08947175601497293, "epoch": 0.4112627986348123, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01593436673283577, "kl": 0.006819352100137621, "learning_rate": 5.303754266211604e-06, "loss": -0.0151, "num_tokens": 76719358.0, "reward": 0.07500000298023224, "reward_std": 0.34990841150283813, "rewards/format_reward/mean": 0.05000000074505806, "rewards/format_reward/std": 0.22072142362594604, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.0870838165283203, "sampling/importance_sampling_ratio/mean": 0.4737929403781891, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3009858131408691, "sampling/sampling_logp_difference/mean": 0.009674138389527798, "step": 723, "step_time": 77.69079896900803 }, { "clip_ratio/high_max": 0.007310531800612807, "clip_ratio/high_mean": 0.002324744942598045, "clip_ratio/low_mean": 0.0026603737787809223, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004985118634067476, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 880.0, "completions/mean_terminated_length": 612.5714721679688, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.12725418899208307, "epoch": 0.4118316268486917, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02654404751956463, "kl": 0.011801481887232512, "learning_rate": 5.298634812286689e-06, "loss": 0.0534, "num_tokens": 76819126.0, "reward": 0.550000011920929, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.806459903717041, "sampling/importance_sampling_ratio/mean": 0.44317975640296936, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0720055103302, "sampling/sampling_logp_difference/mean": 0.013951221480965614, "step": 724, "step_time": 72.6362414532341 }, { "clip_ratio/high_max": 0.01161592424614355, "clip_ratio/high_mean": 0.004215626191580668, "clip_ratio/low_mean": 0.0012195675990369637, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005435193772427738, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 927.0, "completions/mean_length": 618.6000366210938, "completions/mean_terminated_length": 517.25, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.12431852333247662, "epoch": 0.4124004550625711, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013478608801960945, "kl": 0.01399984210729599, "learning_rate": 5.293515358361775e-06, "loss": -0.0447, "num_tokens": 76926670.0, "reward": 1.5750000476837158, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.7749999761581421, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 1.8577808141708374, "sampling/importance_sampling_ratio/mean": 0.390651136636734, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3552327156066895, "sampling/sampling_logp_difference/mean": 0.015594693832099438, "step": 725, "step_time": 73.16176735935733 }, { "clip_ratio/high_max": 0.016119999927468598, "clip_ratio/high_mean": 0.0055031501688063145, "clip_ratio/low_mean": 0.0021569155214820057, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007660065602976829, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 859.625, "completions/mean_terminated_length": 658.7222290039062, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.11230201832950115, "epoch": 0.4129692832764505, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.041247304528951645, "kl": 0.012986995046958327, "learning_rate": 5.28839590443686e-06, "loss": -0.11, "num_tokens": 77034855.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.3084683418273926, "sampling/importance_sampling_ratio/mean": 0.31542715430259705, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1368439197540283, "sampling/sampling_logp_difference/mean": 0.013697466813027859, "step": 726, "step_time": 74.16881672316231 }, { "clip_ratio/high_max": 0.00561705679865554, "clip_ratio/high_mean": 0.0015253751116688363, "clip_ratio/low_mean": 0.0022949218982830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038202969008125365, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 814.5250244140625, "completions/mean_terminated_length": 500.3125, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.08487690007314086, "epoch": 0.41353811149032993, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02948773093521595, "kl": 0.011776736646424979, "learning_rate": 5.283276450511945e-06, "loss": 0.0071, "num_tokens": 77140532.0, "reward": 0.75, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.8812649250030518, "sampling/importance_sampling_ratio/mean": 0.4167528748512268, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.235804557800293, "sampling/sampling_logp_difference/mean": 0.011259041726589203, "step": 727, "step_time": 73.913151914021 }, { "clip_ratio/high_max": 0.011411858256906271, "clip_ratio/high_mean": 0.0027484684251248837, "clip_ratio/low_mean": 0.0020507812732830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0047992497275117785, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 940.9000244140625, "completions/mean_terminated_length": 654.6666870117188, "completions/min_length": 414.0, "completions/min_terminated_length": 414.0, "entropy": 0.11887145880609751, "epoch": 0.4141069397042093, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015253430232405663, "kl": 0.009709474863484502, "learning_rate": 5.278156996587031e-06, "loss": 0.0693, "num_tokens": 77253200.0, "reward": 0.375, "reward_std": 0.7403222322463989, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 1.734257698059082, "sampling/importance_sampling_ratio/mean": 0.3033568263053894, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7747137546539307, "sampling/sampling_logp_difference/mean": 0.0124584399163723, "step": 728, "step_time": 76.97802301426418 }, { "clip_ratio/high_max": 0.012402818305417895, "clip_ratio/high_mean": 0.0033243897632928565, "clip_ratio/low_mean": 0.002880859377910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062052491412032396, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 927.5499877929688, "completions/mean_terminated_length": 638.2000122070312, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.14998515043407679, "epoch": 0.41467576791808874, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013370783068239689, "kl": 0.010835672554094344, "learning_rate": 5.2730375426621165e-06, "loss": 0.0402, "num_tokens": 77359158.0, "reward": 0.4749999940395355, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.0793533325195312, "sampling/importance_sampling_ratio/mean": 0.38197922706604004, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1346755027770996, "sampling/sampling_logp_difference/mean": 0.014506304636597633, "step": 729, "step_time": 72.95409177313559 }, { "clip_ratio/high_max": 0.0075477888458408415, "clip_ratio/high_mean": 0.0018807503365678713, "clip_ratio/low_mean": 0.0008056640799622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026864144165301695, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 854.0, "completions/mean_length": 955.0499877929688, "completions/mean_terminated_length": 630.0, "completions/min_length": 515.0, "completions/min_terminated_length": 515.0, "entropy": 0.11011715792119503, "epoch": 0.4152445961319681, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011333772912621498, "kl": 0.008244496886618435, "learning_rate": 5.267918088737202e-06, "loss": -0.0146, "num_tokens": 77477296.0, "reward": 0.3499999940395355, "reward_std": 0.7696152329444885, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.2745351791381836, "sampling/importance_sampling_ratio/mean": 0.3926575183868408, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.346129894256592, "sampling/sampling_logp_difference/mean": 0.011609146371483803, "step": 730, "step_time": 77.41829308215529 }, { "clip_ratio/high_max": 0.014284842181950808, "clip_ratio/high_mean": 0.005269123270409182, "clip_ratio/low_mean": 0.002102002952597104, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007371126324869692, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 839.375, "completions/mean_terminated_length": 702.9130859375, "completions/min_length": 414.0, "completions/min_terminated_length": 414.0, "entropy": 0.10264689475297928, "epoch": 0.41581342434584756, "frac_reward_zero_std": 0.0, "grad_norm": 0.03752836585044861, "kl": 0.014017586596310139, "learning_rate": 5.2627986348122865e-06, "loss": -0.0284, "num_tokens": 77588199.0, "reward": 1.100000023841858, "reward_std": 0.9818872213363647, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.754197359085083, "sampling/importance_sampling_ratio/mean": 0.5918847322463989, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7623989582061768, "sampling/sampling_logp_difference/mean": 0.012960604391992092, "step": 731, "step_time": 77.2512040596921 }, { "clip_ratio/high_max": 0.008755836810451001, "clip_ratio/high_mean": 0.0024284209503093734, "clip_ratio/low_mean": 0.001367187509458745, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003795608499785885, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 962.0, "completions/mean_length": 937.8500366210938, "completions/mean_terminated_length": 710.727294921875, "completions/min_length": 496.0, "completions/min_terminated_length": 496.0, "entropy": 0.09640469728037715, "epoch": 0.41638225255972694, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.010315167717635632, "kl": 0.009739423519931734, "learning_rate": 5.257679180887372e-06, "loss": -0.0043, "num_tokens": 77699689.0, "reward": 0.4000000059604645, "reward_std": 0.7089175581932068, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.3722617626190186, "sampling/importance_sampling_ratio/mean": 0.38747334480285645, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6852741241455078, "sampling/sampling_logp_difference/mean": 0.011143120005726814, "step": 732, "step_time": 73.80381759023294 }, { "clip_ratio/high_max": 0.014377535146195441, "clip_ratio/high_mean": 0.004444262769538909, "clip_ratio/low_mean": 0.0019989356842415873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006443198421038687, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 871.0, "completions/mean_length": 832.7999877929688, "completions/mean_terminated_length": 574.11767578125, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.10092817666009068, "epoch": 0.4169510807736064, "frac_reward_zero_std": 0.0, "grad_norm": 0.014790006913244724, "kl": 0.012597581138834357, "learning_rate": 5.252559726962457e-06, "loss": 0.0695, "num_tokens": 77791113.0, "reward": 0.824999988079071, "reward_std": 0.9841695427894592, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.3625786304473877, "sampling/importance_sampling_ratio/mean": 0.3717131018638611, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5666534900665283, "sampling/sampling_logp_difference/mean": 0.012450965121388435, "step": 733, "step_time": 69.96819663513452 }, { "clip_ratio/high_max": 0.010817479284014553, "clip_ratio/high_mean": 0.0030141387396724895, "clip_ratio/low_mean": 0.0021240234782453626, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005138162232469767, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 936.5, "completions/mean_terminated_length": 774.0000610351562, "completions/min_length": 493.0, "completions/min_terminated_length": 493.0, "entropy": 0.1063239211216569, "epoch": 0.41751990898748575, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.058187637478113174, "kl": 0.011895747331436723, "learning_rate": 5.247440273037543e-06, "loss": -0.097, "num_tokens": 77888821.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.4124653339385986, "sampling/importance_sampling_ratio/mean": 0.5184426307678223, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.096489191055298, "sampling/sampling_logp_difference/mean": 0.012021643109619617, "step": 734, "step_time": 71.20729206944816 }, { "clip_ratio/high_max": 0.007651610401808284, "clip_ratio/high_mean": 0.0016358072753064334, "clip_ratio/low_mean": 0.0017578124898136593, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00339361974329222, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 954.7250366210938, "completions/mean_terminated_length": 677.625, "completions/min_length": 531.0, "completions/min_terminated_length": 531.0, "entropy": 0.1235187966376543, "epoch": 0.4180887372013652, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03106389380991459, "kl": 0.010265593999065459, "learning_rate": 5.2423208191126274e-06, "loss": -0.0434, "num_tokens": 78002386.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.094015598297119, "sampling/importance_sampling_ratio/mean": 0.5487786531448364, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9216597080230713, "sampling/sampling_logp_difference/mean": 0.013147912919521332, "step": 735, "step_time": 76.60394976218231 }, { "clip_ratio/high_max": 0.0037798354169353843, "clip_ratio/high_mean": 0.0007559670630143955, "clip_ratio/low_mean": 0.0003906250094587449, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011465920724731404, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 941.0, "completions/mean_length": 913.9500122070312, "completions/mean_terminated_length": 623.8181762695312, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "entropy": 0.10858631134033203, "epoch": 0.41865756541524457, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.02042609080672264, "kl": 0.009136207983829081, "learning_rate": 5.237201365187714e-06, "loss": -0.0218, "num_tokens": 78115032.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.461634397506714, "sampling/importance_sampling_ratio/mean": 0.32062020897865295, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1668691635131836, "sampling/sampling_logp_difference/mean": 0.012530053034424782, "step": 736, "step_time": 76.39635455003008 }, { "clip_ratio/high_max": 0.011520650412421674, "clip_ratio/high_mean": 0.003244462044676766, "clip_ratio/low_mean": 0.0022460937398136593, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005490555719006807, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 950.875, "completions/mean_terminated_length": 780.25, "completions/min_length": 546.0, "completions/min_terminated_length": 546.0, "entropy": 0.10595374926924706, "epoch": 0.419226393629124, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024313392117619514, "kl": 0.01160082034766674, "learning_rate": 5.232081911262799e-06, "loss": -0.0413, "num_tokens": 78222427.0, "reward": 0.574999988079071, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.6423332691192627, "sampling/importance_sampling_ratio/mean": 0.4149478077888489, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.991543173789978, "sampling/sampling_logp_difference/mean": 0.01248177420347929, "step": 737, "step_time": 74.08252290333621 }, { "clip_ratio/high_max": 0.014304490410722792, "clip_ratio/high_mean": 0.0065692198113538325, "clip_ratio/low_mean": 0.002148437533833203, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.008717657357919961, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 816.5, "completions/mean_terminated_length": 663.1304321289062, "completions/min_length": 371.0, "completions/min_terminated_length": 371.0, "entropy": 0.13244170602411032, "epoch": 0.4197952218430034, "frac_reward_zero_std": 0.0, "grad_norm": 0.038092661648988724, "kl": 0.01222924911417067, "learning_rate": 5.226962457337885e-06, "loss": 0.0112, "num_tokens": 78329775.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.295356273651123, "sampling/importance_sampling_ratio/mean": 0.37303727865219116, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8390698432922363, "sampling/sampling_logp_difference/mean": 0.014559012837707996, "step": 738, "step_time": 73.85933840903454 }, { "clip_ratio/high_max": 0.012237452785484493, "clip_ratio/high_mean": 0.003462732958723791, "clip_ratio/low_mean": 0.0018457947371643968, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0053085275867488235, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 853.0, "completions/mean_length": 794.8500366210938, "completions/mean_terminated_length": 565.7000122070312, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.11273576878011227, "epoch": 0.4203640500568828, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02935555949807167, "kl": 0.011061444412916899, "learning_rate": 5.221843003412969e-06, "loss": -0.0578, "num_tokens": 78433737.0, "reward": 0.925000011920929, "reward_std": 0.9710555076599121, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.874622106552124, "sampling/importance_sampling_ratio/mean": 0.4702284336090088, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8934834003448486, "sampling/sampling_logp_difference/mean": 0.01302949246019125, "step": 739, "step_time": 75.63228158699349 }, { "clip_ratio/high_max": 0.01181427575647831, "clip_ratio/high_mean": 0.0038906978734303266, "clip_ratio/low_mean": 0.0012908070275443606, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005181504791835323, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 844.0250244140625, "completions/mean_terminated_length": 664.0499877929688, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.08393150614574552, "epoch": 0.42093287827076226, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0272999070584774, "kl": 0.010881674708798528, "learning_rate": 5.216723549488055e-06, "loss": -0.1081, "num_tokens": 78541170.0, "reward": 0.75, "reward_std": 0.8697184920310974, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.839195966720581, "sampling/importance_sampling_ratio/mean": 0.4983806610107422, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.498704195022583, "sampling/sampling_logp_difference/mean": 0.01102385949343443, "step": 740, "step_time": 71.9544981259387 }, { "clip_ratio/high_max": 0.010013948893174529, "clip_ratio/high_mean": 0.0030100640360615216, "clip_ratio/low_mean": 0.0018554687339928932, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004865532828262076, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 789.9000244140625, "completions/mean_terminated_length": 598.3636474609375, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.12581412866711617, "epoch": 0.42150170648464164, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027480555698275566, "kl": 0.01160855550551787, "learning_rate": 5.21160409556314e-06, "loss": 0.0101, "num_tokens": 78642526.0, "reward": 1.0, "reward_std": 0.9607689380645752, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.313972234725952, "sampling/importance_sampling_ratio/mean": 0.3858979344367981, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7851698398590088, "sampling/sampling_logp_difference/mean": 0.01401616632938385, "step": 741, "step_time": 69.98270298889838 }, { "clip_ratio/high_max": 0.006872706580907106, "clip_ratio/high_mean": 0.0013745413307333365, "clip_ratio/low_mean": 0.0022216796642169356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035962209512945265, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 864.0, "completions/mean_length": 788.2750244140625, "completions/mean_terminated_length": 552.5499877929688, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.12580394558608532, "epoch": 0.42207053469852107, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011169393546879292, "kl": 0.01327926164958626, "learning_rate": 5.206484641638225e-06, "loss": 0.0177, "num_tokens": 78751433.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.6890759468078613, "sampling/importance_sampling_ratio/mean": 0.3012528717517853, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6750949621200562, "sampling/sampling_logp_difference/mean": 0.016240956261754036, "step": 742, "step_time": 75.70054221525788 }, { "clip_ratio/high_max": 0.01304739830084145, "clip_ratio/high_mean": 0.00499092650716193, "clip_ratio/low_mean": 0.002050781244179234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007041707896860316, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 836.0, "completions/mean_length": 795.0750122070312, "completions/mean_terminated_length": 542.0526123046875, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.09115746337920427, "epoch": 0.42263936291240045, "frac_reward_zero_std": 0.0, "grad_norm": 0.024852190166711807, "kl": 0.010426352208014578, "learning_rate": 5.201365187713311e-06, "loss": 0.0211, "num_tokens": 78857828.0, "reward": 0.949999988079071, "reward_std": 1.0114725828170776, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.2406103610992432, "sampling/importance_sampling_ratio/mean": 0.31477728486061096, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0742204189300537, "sampling/sampling_logp_difference/mean": 0.012101059779524803, "step": 743, "step_time": 73.61900552501902 }, { "clip_ratio/high_max": 0.011452261067461222, "clip_ratio/high_mean": 0.003621873998781666, "clip_ratio/low_mean": 0.003320312465802999, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006942186446394771, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 917.0, "completions/mean_length": 910.125, "completions/mean_terminated_length": 673.6154174804688, "completions/min_length": 467.0, "completions/min_terminated_length": 467.0, "entropy": 0.1500197807326913, "epoch": 0.4232081911262799, "frac_reward_zero_std": 0.0, "grad_norm": 0.02108066901564598, "kl": 0.014209682296495885, "learning_rate": 5.196245733788396e-06, "loss": -0.0695, "num_tokens": 78972233.0, "reward": 0.6000000238418579, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.562946081161499, "sampling/importance_sampling_ratio/mean": 0.2240275889635086, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.957193374633789, "sampling/sampling_logp_difference/mean": 0.014919663779437542, "step": 744, "step_time": 75.81464793905616 }, { "clip_ratio/high_max": 0.014973363082390279, "clip_ratio/high_mean": 0.004054410630487837, "clip_ratio/low_mean": 0.0010742187569121597, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0051286292873555794, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 807.9750366210938, "completions/mean_terminated_length": 663.9583740234375, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.11558949295431376, "epoch": 0.42377701934015927, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01764707826077938, "kl": 0.010264973621815443, "learning_rate": 5.191126279863482e-06, "loss": -0.0122, "num_tokens": 79070584.0, "reward": 1.2000000476837158, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.3047237396240234, "sampling/importance_sampling_ratio/mean": 0.3940344452857971, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.271893262863159, "sampling/sampling_logp_difference/mean": 0.013455450534820557, "step": 745, "step_time": 70.90019814413972 }, { "clip_ratio/high_max": 0.005912020453251898, "clip_ratio/high_mean": 0.0014863554533803836, "clip_ratio/low_mean": 0.002456737398460973, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003943092830013484, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 974.9750366210938, "completions/mean_terminated_length": 806.1111450195312, "completions/min_length": 644.0, "completions/min_terminated_length": 644.0, "entropy": 0.16126598417758942, "epoch": 0.4243458475540387, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013175169005990028, "kl": 0.011136185028590262, "learning_rate": 5.186006825938566e-06, "loss": 0.078, "num_tokens": 79171447.0, "reward": 0.30000001192092896, "reward_std": 0.6076436042785645, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290207743644714, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.9736547470092773, "sampling/importance_sampling_ratio/mean": 0.41548028588294983, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.389625906944275, "sampling/sampling_logp_difference/mean": 0.015674900263547897, "step": 746, "step_time": 74.79182602395304 }, { "clip_ratio/high_max": 0.009031643916387111, "clip_ratio/high_mean": 0.002935330572654493, "clip_ratio/low_mean": 0.0023016741033643484, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005237004574155435, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 924.0, "completions/mean_length": 736.9500122070312, "completions/mean_terminated_length": 524.7825927734375, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.14289200957864523, "epoch": 0.4249146757679181, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01884675957262516, "kl": 0.011503520770929754, "learning_rate": 5.180887372013652e-06, "loss": -0.0418, "num_tokens": 79273573.0, "reward": 0.8500000238418579, "reward_std": 0.8638019561767578, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.3837838172912598, "sampling/importance_sampling_ratio/mean": 0.35266801714897156, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6666829586029053, "sampling/sampling_logp_difference/mean": 0.016747256740927696, "step": 747, "step_time": 71.29471813002601 }, { "clip_ratio/high_max": 0.008845838659908623, "clip_ratio/high_mean": 0.0019264004658907652, "clip_ratio/low_mean": 0.00222167971151066, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00414808010100387, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 882.0, "completions/mean_length": 764.9500122070312, "completions/mean_terminated_length": 553.0, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.11860990338027477, "epoch": 0.4254835039817975, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.034372832626104355, "kl": 0.014897263376042247, "learning_rate": 5.175767918088737e-06, "loss": 0.0033, "num_tokens": 79377579.0, "reward": 0.925000011920929, "reward_std": 0.9167249798774719, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.877023696899414, "sampling/importance_sampling_ratio/mean": 0.4785333573818207, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4079747200012207, "sampling/sampling_logp_difference/mean": 0.014798969030380249, "step": 748, "step_time": 72.01728898123838 }, { "clip_ratio/high_max": 0.014027222408913076, "clip_ratio/high_mean": 0.005066801386419684, "clip_ratio/low_mean": 0.0016467385285068303, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0067135399440303445, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 866.0, "completions/mean_length": 719.4249877929688, "completions/mean_terminated_length": 572.7777709960938, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.1013598358258605, "epoch": 0.4260523321956769, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02554631233215332, "kl": 0.016515010152943432, "learning_rate": 5.170648464163823e-06, "loss": -0.0536, "num_tokens": 79471044.0, "reward": 1.100000023841858, "reward_std": 0.8711913824081421, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.0460073947906494, "sampling/importance_sampling_ratio/mean": 0.35569173097610474, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.390432119369507, "sampling/sampling_logp_difference/mean": 0.013068403117358685, "step": 749, "step_time": 71.11516511882655 }, { "clip_ratio/high_max": 0.013955974078271538, "clip_ratio/high_mean": 0.004108656328753568, "clip_ratio/low_mean": 0.0017822265945142135, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005890882865060121, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 934.0, "completions/mean_length": 849.875, "completions/mean_terminated_length": 588.6875, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.10824267566204071, "epoch": 0.42662116040955633, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.025913432240486145, "kl": 0.009630644461140037, "learning_rate": 5.165529010238907e-06, "loss": -0.0679, "num_tokens": 79580575.0, "reward": 0.699999988079071, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.9881900548934937, "sampling/importance_sampling_ratio/mean": 0.3673827350139618, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9904546737670898, "sampling/sampling_logp_difference/mean": 0.013005665503442287, "step": 750, "step_time": 73.97669573524036 }, { "clip_ratio/high_max": 0.013419174938462675, "clip_ratio/high_mean": 0.004098876874195412, "clip_ratio/low_mean": 0.002221679678768851, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0063205566839315, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 888.9249877929688, "completions/mean_terminated_length": 638.0714721679688, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.1220629122108221, "epoch": 0.4271899886234357, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04351399838924408, "kl": 0.016616859706118703, "learning_rate": 5.1604095563139935e-06, "loss": -0.062, "num_tokens": 79679836.0, "reward": 0.6500000357627869, "reward_std": 0.9212588667869568, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.232412576675415, "sampling/importance_sampling_ratio/mean": 0.3561459183692932, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.4398722648620605, "sampling/sampling_logp_difference/mean": 0.012807980179786682, "step": 751, "step_time": 75.33093628683127 }, { "clip_ratio/high_max": 0.010774850728921592, "clip_ratio/high_mean": 0.0021549701123149134, "clip_ratio/low_mean": 0.0010253906439174898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0031803607562324032, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 799.6500244140625, "completions/mean_terminated_length": 425.7333679199219, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.07028470048680902, "epoch": 0.42775881683731515, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.019133547320961952, "kl": 0.010254351189360023, "learning_rate": 5.155290102389079e-06, "loss": -0.0484, "num_tokens": 79784462.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.8666120767593384, "sampling/importance_sampling_ratio/mean": 0.406419038772583, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.095726490020752, "sampling/sampling_logp_difference/mean": 0.0104867834597826, "step": 752, "step_time": 72.41496241721325 }, { "clip_ratio/high_max": 0.011917851283214986, "clip_ratio/high_mean": 0.003623942917329259, "clip_ratio/low_mean": 0.0034179687791038305, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.007041911710985005, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 860.0, "completions/mean_length": 889.8250122070312, "completions/mean_terminated_length": 576.75, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.10692128352820873, "epoch": 0.4283276450511945, "frac_reward_zero_std": 0.0, "grad_norm": 0.035393960773944855, "kl": 0.025628606439568102, "learning_rate": 5.150170648464164e-06, "loss": 0.0107, "num_tokens": 79893479.0, "reward": 0.574999988079071, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.52571702003479, "sampling/importance_sampling_ratio/mean": 0.3218558430671692, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.806556463241577, "sampling/sampling_logp_difference/mean": 0.012267185375094414, "step": 753, "step_time": 72.86081936326809 }, { "clip_ratio/high_max": 0.009538626240100712, "clip_ratio/high_mean": 0.0027894548984477296, "clip_ratio/low_mean": 0.0012013402592856437, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003990795172285289, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 664.2999877929688, "completions/mean_terminated_length": 491.1111145019531, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.12237957213073969, "epoch": 0.42889647326507396, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03705206140875816, "kl": 0.013422232703305781, "learning_rate": 5.145051194539249e-06, "loss": 0.0978, "num_tokens": 79987955.0, "reward": 1.3250000476837158, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.2343215942382812, "sampling/importance_sampling_ratio/mean": 0.4538145959377289, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9713797569274902, "sampling/sampling_logp_difference/mean": 0.015133947134017944, "step": 754, "step_time": 68.80319073004648 }, { "clip_ratio/high_max": 0.010046439914731309, "clip_ratio/high_mean": 0.0027782149772974662, "clip_ratio/low_mean": 0.0011230468771827873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390126186539419, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 935.0, "completions/mean_length": 702.4750366210938, "completions/mean_terminated_length": 580.5172119140625, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.08937626797705889, "epoch": 0.42946530147895334, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.017970433458685875, "kl": 0.01713444327469915, "learning_rate": 5.139931740614334e-06, "loss": -0.0279, "num_tokens": 80079502.0, "reward": 1.4500000476837158, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.7250000238418579, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.9414212703704834, "sampling/importance_sampling_ratio/mean": 0.43447884917259216, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.25662899017334, "sampling/sampling_logp_difference/mean": 0.014294209890067577, "step": 755, "step_time": 69.97815450327471 }, { "clip_ratio/high_max": 0.012130277696996927, "clip_ratio/high_mean": 0.0037512374110519886, "clip_ratio/low_mean": 0.00244140625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006192643661051989, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 909.5250244140625, "completions/mean_terminated_length": 696.9285888671875, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.11483260709792376, "epoch": 0.4300341296928328, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04003571346402168, "kl": 0.012847697013057768, "learning_rate": 5.13481228668942e-06, "loss": -0.0876, "num_tokens": 80190619.0, "reward": 0.699999988079071, "reward_std": 0.9660917520523071, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.1514267921447754, "sampling/importance_sampling_ratio/mean": 0.48146551847457886, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.216493844985962, "sampling/sampling_logp_difference/mean": 0.013300671242177486, "step": 756, "step_time": 75.72672708681785 }, { "clip_ratio/high_max": 0.007547096349298954, "clip_ratio/high_mean": 0.00285288855957333, "clip_ratio/low_mean": 0.00205853010629653, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.004911418625852093, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 940.0, "completions/mean_length": 852.4249877929688, "completions/mean_terminated_length": 595.0625, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.10544235538691282, "epoch": 0.43060295790671216, "frac_reward_zero_std": 0.0, "grad_norm": 0.01407522615045309, "kl": 0.011195081926416606, "learning_rate": 5.129692832764505e-06, "loss": 0.0312, "num_tokens": 80294604.0, "reward": 0.6500000357627869, "reward_std": 0.8638019561767578, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.6075999736785889, "sampling/importance_sampling_ratio/mean": 0.3161342740058899, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8278077840805054, "sampling/sampling_logp_difference/mean": 0.011833997443318367, "step": 757, "step_time": 73.37859963136725 }, { "clip_ratio/high_max": 0.011905910447239876, "clip_ratio/high_mean": 0.003063844662392512, "clip_ratio/low_mean": 0.0018066406264551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004870485223364085, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 831.0, "completions/mean_terminated_length": 688.3478393554688, "completions/min_length": 450.0, "completions/min_terminated_length": 450.0, "entropy": 0.15367123670876026, "epoch": 0.4311717861205916, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.007135201711207628, "kl": 0.012511403067037463, "learning_rate": 5.12457337883959e-06, "loss": -0.0055, "num_tokens": 80396620.0, "reward": 1.0500000715255737, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.393320083618164, "sampling/importance_sampling_ratio/mean": 0.4239901900291443, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.216026544570923, "sampling/sampling_logp_difference/mean": 0.015842674300074577, "step": 758, "step_time": 72.61126731499098 }, { "clip_ratio/high_max": 0.012392172066029161, "clip_ratio/high_mean": 0.003303028250229545, "clip_ratio/low_mean": 0.001977539053768851, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005280567303998396, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 769.5, "completions/mean_terminated_length": 539.2380981445312, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.11703892145305872, "epoch": 0.431740614334471, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024219151586294174, "kl": 0.013793755555525422, "learning_rate": 5.119453924914676e-06, "loss": 0.1287, "num_tokens": 80501856.0, "reward": 1.024999976158142, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.3478145599365234, "sampling/importance_sampling_ratio/mean": 0.4638550281524658, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.132826328277588, "sampling/sampling_logp_difference/mean": 0.014774873852729797, "step": 759, "step_time": 70.61028464045376 }, { "clip_ratio/high_max": 0.01010917971143499, "clip_ratio/high_mean": 0.0033728918060660362, "clip_ratio/low_mean": 0.0022460937416326487, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005618985538603738, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 852.9000244140625, "completions/mean_terminated_length": 643.7777709960938, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.09056534664705396, "epoch": 0.4323094425483504, "frac_reward_zero_std": 0.0, "grad_norm": 0.03972447291016579, "kl": 0.03701018140418455, "learning_rate": 5.114334470989762e-06, "loss": 0.1617, "num_tokens": 80603620.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9889328479766846, "sampling/importance_sampling_ratio/mean": 0.4448617100715637, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.088919162750244, "sampling/sampling_logp_difference/mean": 0.01161857508122921, "step": 760, "step_time": 73.01987035549246 }, { "clip_ratio/high_max": 0.017968595726415515, "clip_ratio/high_mean": 0.006386710447259247, "clip_ratio/low_mean": 0.002221679675130872, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0086083899368532, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 797.3500366210938, "completions/mean_terminated_length": 570.7000122070312, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.10187135171145201, "epoch": 0.4328782707622298, "frac_reward_zero_std": 0.0, "grad_norm": 0.027599867433309555, "kl": 0.014397892518900335, "learning_rate": 5.109215017064847e-06, "loss": 0.0227, "num_tokens": 80706466.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.507584810256958, "sampling/importance_sampling_ratio/mean": 0.5112188458442688, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2357611656188965, "sampling/sampling_logp_difference/mean": 0.012406869791448116, "step": 761, "step_time": 70.79680492985062 }, { "clip_ratio/high_max": 0.013328326283954084, "clip_ratio/high_mean": 0.004969804955180734, "clip_ratio/low_mean": 0.0020154976809863, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006985302723478526, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 902.0, "completions/mean_length": 815.9750366210938, "completions/mean_terminated_length": 662.2174072265625, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.11485059652477503, "epoch": 0.4334470989761092, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018286656588315964, "kl": 0.012137348763644695, "learning_rate": 5.104095563139932e-06, "loss": 0.0511, "num_tokens": 80798561.0, "reward": 1.0500000715255737, "reward_std": 0.9594335556030273, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.366629958152771, "sampling/importance_sampling_ratio/mean": 0.2934208810329437, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6589784622192383, "sampling/sampling_logp_difference/mean": 0.01393929310142994, "step": 762, "step_time": 71.395089937374 }, { "clip_ratio/high_max": 0.011245330853853375, "clip_ratio/high_mean": 0.002875294245313853, "clip_ratio/low_mean": 0.0005126953219587449, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033879895781865343, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 852.0, "completions/mean_length": 693.5250244140625, "completions/mean_terminated_length": 515.5769653320312, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.08113086596131325, "epoch": 0.4340159271899886, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.016091974452137947, "kl": 0.012494117021560669, "learning_rate": 5.098976109215017e-06, "loss": 0.0513, "num_tokens": 80905038.0, "reward": 1.3000000715255737, "reward_std": 0.9660918712615967, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.9557551145553589, "sampling/importance_sampling_ratio/mean": 0.39493051171302795, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3204753398895264, "sampling/sampling_logp_difference/mean": 0.01281064748764038, "step": 763, "step_time": 71.99789680191316 }, { "clip_ratio/high_max": 0.015101949102245271, "clip_ratio/high_mean": 0.004305746493628249, "clip_ratio/low_mean": 0.002349609392695129, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006655355799011886, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 865.0, "completions/mean_length": 781.5499877929688, "completions/mean_terminated_length": 513.5789794921875, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.07611763570457697, "epoch": 0.43458475540386804, "frac_reward_zero_std": 0.0, "grad_norm": 0.02950160950422287, "kl": 0.009619193326216191, "learning_rate": 5.0938566552901025e-06, "loss": 0.0705, "num_tokens": 81011116.0, "reward": 0.7750000357627869, "reward_std": 0.8911963105201721, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.0541279315948486, "sampling/importance_sampling_ratio/mean": 0.41103821992874146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3106606006622314, "sampling/sampling_logp_difference/mean": 0.010449530556797981, "step": 764, "step_time": 73.83704148419201 }, { "clip_ratio/high_max": 0.008575037005357444, "clip_ratio/high_mean": 0.002248342483653687, "clip_ratio/low_mean": 0.001782226565410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004030569107271731, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 868.6749877929688, "completions/mean_terminated_length": 658.5294189453125, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.10846410319209099, "epoch": 0.4351535836177474, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011125943623483181, "kl": 0.010163747821934521, "learning_rate": 5.088737201365188e-06, "loss": 0.0064, "num_tokens": 81124999.0, "reward": 0.7750000357627869, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.6287847757339478, "sampling/importance_sampling_ratio/mean": 0.23379652202129364, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4727225303649902, "sampling/sampling_logp_difference/mean": 0.01323630940169096, "step": 765, "step_time": 75.47296350286342 }, { "clip_ratio/high_max": 0.008680669008754194, "clip_ratio/high_mean": 0.0021838041066075675, "clip_ratio/low_mean": 0.0011718749992724042, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033556790585862473, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 881.0750122070312, "completions/mean_terminated_length": 615.6428833007812, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.11381140910089016, "epoch": 0.43572241183162685, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.009395127184689045, "kl": 0.01027832500403747, "learning_rate": 5.0836177474402725e-06, "loss": 0.0359, "num_tokens": 81225730.0, "reward": 0.4749999940395355, "reward_std": 0.715667188167572, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.8323538303375244, "sampling/importance_sampling_ratio/mean": 0.4274255335330963, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.236043930053711, "sampling/sampling_logp_difference/mean": 0.012869032099843025, "step": 766, "step_time": 70.6522734221071 }, { "clip_ratio/high_max": 0.007088673068210483, "clip_ratio/high_mean": 0.0022551520087290555, "clip_ratio/low_mean": 0.0011718750101863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003427027026191354, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 970.0, "completions/mean_length": 808.9750366210938, "completions/mean_terminated_length": 614.4285888671875, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.12120199203491211, "epoch": 0.43629124004550623, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.032431308180093765, "kl": 0.01245894085150212, "learning_rate": 5.078498293515359e-06, "loss": -0.0434, "num_tokens": 81329129.0, "reward": 0.925000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.5164215564727783, "sampling/importance_sampling_ratio/mean": 0.5113648176193237, "sampling/importance_sampling_ratio/min": 0.01598995365202427, "sampling/sampling_logp_difference/max": 2.02677583694458, "sampling/sampling_logp_difference/mean": 0.01360438484698534, "step": 767, "step_time": 72.89062108215876 }, { "clip_ratio/high_max": 0.01680547243449837, "clip_ratio/high_mean": 0.007783354725688696, "clip_ratio/low_mean": 0.0013517855040845461, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00913513998966664, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 724.5, "completions/mean_terminated_length": 610.8965454101562, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.10438387840986252, "epoch": 0.43686006825938567, "frac_reward_zero_std": 0.0, "grad_norm": 0.023963814601302147, "kl": 0.01592353341402486, "learning_rate": 5.073378839590444e-06, "loss": -0.055, "num_tokens": 81432605.0, "reward": 1.399999976158142, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.637833833694458, "sampling/importance_sampling_ratio/mean": 0.4046114981174469, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4620485305786133, "sampling/sampling_logp_difference/mean": 0.01394722145050764, "step": 768, "step_time": 72.97964398306794 }, { "clip_ratio/high_max": 0.0072922855033539236, "clip_ratio/high_mean": 0.002136575640179217, "clip_ratio/low_mean": 0.002294921869179234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00443149745115079, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 863.0, "completions/mean_length": 826.2250366210938, "completions/mean_terminated_length": 584.5, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.09691999666392803, "epoch": 0.43742889647326505, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.044913098216056824, "kl": 0.011307016480714083, "learning_rate": 5.06825938566553e-06, "loss": -0.0521, "num_tokens": 81525998.0, "reward": 0.75, "reward_std": 0.8987170457839966, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.5308475494384766, "sampling/importance_sampling_ratio/mean": 0.5450013875961304, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.636375904083252, "sampling/sampling_logp_difference/mean": 0.012094097211956978, "step": 769, "step_time": 73.00607784488238 }, { "clip_ratio/high_max": 0.012020998008665629, "clip_ratio/high_mean": 0.003085272815951612, "clip_ratio/low_mean": 0.0015624999832652975, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0046477727591991425, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 871.0, "completions/mean_length": 934.9249877929688, "completions/mean_terminated_length": 628.1111450195312, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.1038835970684886, "epoch": 0.4379977246871445, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.008800541050732136, "kl": 0.00914976914646104, "learning_rate": 5.063139931740614e-06, "loss": 0.0383, "num_tokens": 81640203.0, "reward": 0.42500001192092896, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.41927170753479, "sampling/importance_sampling_ratio/mean": 0.32844269275665283, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.316375732421875, "sampling/sampling_logp_difference/mean": 0.011809231713414192, "step": 770, "step_time": 76.22769441781566 }, { "clip_ratio/high_max": 0.013132020947523415, "clip_ratio/high_mean": 0.004168939020019025, "clip_ratio/low_mean": 0.0027779062802437693, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.0069468452711589634, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 876.5250244140625, "completions/mean_terminated_length": 677.0, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.1529801981523633, "epoch": 0.43856655290102387, "frac_reward_zero_std": 0.0, "grad_norm": 0.03558359295129776, "kl": 0.012256759742740542, "learning_rate": 5.0580204778157e-06, "loss": -0.1674, "num_tokens": 81752600.0, "reward": 0.7750000357627869, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.011303424835205, "sampling/importance_sampling_ratio/mean": 0.3824477195739746, "sampling/importance_sampling_ratio/min": 0.006587297655642033, "sampling/sampling_logp_difference/max": 1.54129958152771, "sampling/sampling_logp_difference/mean": 0.01559287030249834, "step": 771, "step_time": 78.31738133658655 }, { "clip_ratio/high_max": 0.004457229631952941, "clip_ratio/high_mean": 0.0008914459467632696, "clip_ratio/low_mean": 0.0007812500098225428, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016726959383959183, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 985.4500122070312, "completions/mean_terminated_length": 638.5, "completions/min_length": 481.0, "completions/min_terminated_length": 481.0, "entropy": 0.07856439985334873, "epoch": 0.4391353811149033, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.00994076021015644, "kl": 0.008527884085197002, "learning_rate": 5.052901023890785e-06, "loss": 0.0272, "num_tokens": 81856466.0, "reward": 0.15000000596046448, "reward_std": 0.48304590582847595, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.05000000074505806, "rewards/gpt_judge_reward/std": 0.22072142362594604, "sampling/importance_sampling_ratio/max": 1.9092004299163818, "sampling/importance_sampling_ratio/mean": 0.4041643738746643, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9445834159851074, "sampling/sampling_logp_difference/mean": 0.00929373037070036, "step": 772, "step_time": 71.52251285198145 }, { "clip_ratio/high_max": 0.012210739019792527, "clip_ratio/high_mean": 0.0036801823880523443, "clip_ratio/low_mean": 0.0018310546620341484, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005511237017344683, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 897.0, "completions/mean_length": 847.0750122070312, "completions/mean_terminated_length": 552.2000122070312, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.0941723482683301, "epoch": 0.4397042093287827, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03989388421177864, "kl": 0.010401375591754913, "learning_rate": 5.0477815699658705e-06, "loss": -0.0442, "num_tokens": 81961781.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.9461047649383545, "sampling/importance_sampling_ratio/mean": 0.4641284942626953, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.704646587371826, "sampling/sampling_logp_difference/mean": 0.011564284563064575, "step": 773, "step_time": 72.11561779608019 }, { "clip_ratio/high_max": 0.011383647157344967, "clip_ratio/high_mean": 0.0037747882888652384, "clip_ratio/low_mean": 0.0028564453386934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006631233525695279, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 924.0, "completions/mean_length": 902.6749877929688, "completions/mean_terminated_length": 650.6923217773438, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.15152319055050611, "epoch": 0.4402730375426621, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024427197873592377, "kl": 0.013903883867897093, "learning_rate": 5.042662116040955e-06, "loss": -0.0589, "num_tokens": 82065664.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.242771625518799, "sampling/importance_sampling_ratio/mean": 0.33917927742004395, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.235938787460327, "sampling/sampling_logp_difference/mean": 0.01589054800570011, "step": 774, "step_time": 72.79363338672556 }, { "clip_ratio/high_max": 0.011486229137517512, "clip_ratio/high_mean": 0.00349263587850146, "clip_ratio/low_mean": 0.002197265628637979, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005689901416189969, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 944.0, "completions/mean_length": 913.7750244140625, "completions/mean_terminated_length": 656.5833740234375, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "entropy": 0.09012577403336763, "epoch": 0.44084186575654155, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019826043397188187, "kl": 0.013210127945058048, "learning_rate": 5.037542662116041e-06, "loss": -0.0597, "num_tokens": 82171871.0, "reward": 0.45000001788139343, "reward_std": 0.7828285098075867, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.6505521535873413, "sampling/importance_sampling_ratio/mean": 0.27969393134117126, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1026694774627686, "sampling/sampling_logp_difference/mean": 0.011379430070519447, "step": 775, "step_time": 72.68622471601702 }, { "clip_ratio/high_max": 0.008015615807380527, "clip_ratio/high_mean": 0.0017037267753039487, "clip_ratio/low_mean": 0.0015380859549622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032418127811979502, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 913.0, "completions/mean_length": 913.0, "completions/mean_terminated_length": 469.0, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.12055742181837559, "epoch": 0.44141069397042093, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025557538494467735, "kl": 0.012282780138775706, "learning_rate": 5.032423208191127e-06, "loss": -0.0448, "num_tokens": 82284927.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.6786309480667114, "sampling/importance_sampling_ratio/mean": 0.3676047921180725, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1921138763427734, "sampling/sampling_logp_difference/mean": 0.012824914418160915, "step": 776, "step_time": 77.23428576276638 }, { "clip_ratio/high_max": 0.0013631406472995877, "clip_ratio/high_mean": 0.00027262812363915145, "clip_ratio/low_mean": 0.00120902867638506, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014816568000242114, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 917.0, "completions/mean_length": 979.7999877929688, "completions/mean_terminated_length": 803.0, "completions/min_length": 733.0, "completions/min_terminated_length": 733.0, "entropy": 0.09092570282518864, "epoch": 0.44197952218430037, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.008964302018284798, "kl": 0.008096031902823597, "learning_rate": 5.0273037542662114e-06, "loss": 0.0341, "num_tokens": 82386887.0, "reward": 0.22500000894069672, "reward_std": 0.47971677780151367, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.7345852851867676, "sampling/importance_sampling_ratio/mean": 0.5011362433433533, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2782957553863525, "sampling/sampling_logp_difference/mean": 0.01106193196028471, "step": 777, "step_time": 71.69396719778888 }, { "clip_ratio/high_max": 0.010827984660863876, "clip_ratio/high_mean": 0.0031680392276030034, "clip_ratio/low_mean": 0.002734374997089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005902414210140705, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 918.0499877929688, "completions/mean_terminated_length": 638.727294921875, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.09667542250826955, "epoch": 0.44254835039817975, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.010204477235674858, "kl": 0.009750127966981381, "learning_rate": 5.022184300341297e-06, "loss": 0.0494, "num_tokens": 82493577.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.670924186706543, "sampling/importance_sampling_ratio/mean": 0.38330531120300293, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0971760749816895, "sampling/sampling_logp_difference/mean": 0.011696582660079002, "step": 778, "step_time": 76.78302929527126 }, { "clip_ratio/high_max": 0.013073840353172272, "clip_ratio/high_mean": 0.003943297837395221, "clip_ratio/low_mean": 0.0017562254070071504, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005699523113435134, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 962.0, "completions/mean_length": 823.6500244140625, "completions/mean_terminated_length": 623.2999877929688, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.09137897985056043, "epoch": 0.4431171786120592, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01367301307618618, "kl": 0.010929677227977663, "learning_rate": 5.017064846416382e-06, "loss": 0.048, "num_tokens": 82586659.0, "reward": 0.8500000238418579, "reward_std": 0.9212588667869568, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.9080922603607178, "sampling/importance_sampling_ratio/mean": 0.39801087975502014, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0977399349212646, "sampling/sampling_logp_difference/mean": 0.011793999932706356, "step": 779, "step_time": 72.03563842689618 }, { "clip_ratio/high_max": 0.015413929941132665, "clip_ratio/high_mean": 0.00493002294388134, "clip_ratio/low_mean": 0.002006288319535088, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006936311139725149, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 912.5750122070312, "completions/mean_terminated_length": 745.4375, "completions/min_length": 428.0, "completions/min_terminated_length": 428.0, "entropy": 0.12840588949620724, "epoch": 0.44368600682593856, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01407639216631651, "kl": 0.010064221045468003, "learning_rate": 5.011945392491468e-06, "loss": 0.0255, "num_tokens": 82695610.0, "reward": 0.6500000357627869, "reward_std": 0.8638019561767578, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.3839325904846191, "sampling/importance_sampling_ratio/mean": 0.3457905352115631, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6122546195983887, "sampling/sampling_logp_difference/mean": 0.014091558754444122, "step": 780, "step_time": 76.05903872754425 }, { "clip_ratio/high_max": 0.013153973210137337, "clip_ratio/high_mean": 0.003964156610891223, "clip_ratio/low_mean": 0.000903320327779511, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004867476964136586, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 596.0, "completions/mean_length": 837.7000122070312, "completions/mean_terminated_length": 450.7692565917969, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.13902926910668612, "epoch": 0.444254835039818, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03151074796915054, "kl": 0.012778540141880512, "learning_rate": 5.006825938566552e-06, "loss": 0.0332, "num_tokens": 82807286.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.5519821643829346, "sampling/importance_sampling_ratio/mean": 0.5179317593574524, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.299182176589966, "sampling/sampling_logp_difference/mean": 0.014324890449643135, "step": 781, "step_time": 74.92272503930144 }, { "clip_ratio/high_max": 0.010981232160702348, "clip_ratio/high_mean": 0.002313342469278723, "clip_ratio/low_mean": 0.0006347656344587449, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029481080564437434, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 861.0, "completions/mean_length": 665.2250366210938, "completions/mean_terminated_length": 561.0645141601562, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.09456228464841843, "epoch": 0.4448236632536974, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01619098149240017, "kl": 0.012673899298533797, "learning_rate": 5.001706484641638e-06, "loss": -0.0066, "num_tokens": 82901655.0, "reward": 1.5500000715255737, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.7749999761581421, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 1.6243140697479248, "sampling/importance_sampling_ratio/mean": 0.3375782370567322, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8749711513519287, "sampling/sampling_logp_difference/mean": 0.013332106173038483, "step": 782, "step_time": 66.19661697512493 }, { "clip_ratio/high_max": 0.015329674119129777, "clip_ratio/high_mean": 0.006006205687299371, "clip_ratio/low_mean": 0.000903320327779511, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006909526011440903, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 939.0, "completions/mean_length": 690.9249877929688, "completions/mean_terminated_length": 579.9000244140625, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.10105546284466982, "epoch": 0.4453924914675768, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.012302327901124954, "kl": 0.010353070450946689, "learning_rate": 4.996587030716724e-06, "loss": -0.0528, "num_tokens": 83006508.0, "reward": 1.5, "reward_std": 0.8770580291748047, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.75, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.2081853151321411, "sampling/importance_sampling_ratio/mean": 0.24627690017223358, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.071833372116089, "sampling/sampling_logp_difference/mean": 0.013919288292527199, "step": 783, "step_time": 72.14108768547885 }, { "clip_ratio/high_max": 0.013200241141021252, "clip_ratio/high_mean": 0.003751900716451928, "clip_ratio/low_mean": 0.001967875476111658, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005719776003388688, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 738.9000244140625, "completions/mean_terminated_length": 567.8399658203125, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.12141419854015112, "epoch": 0.4459613196814562, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02184910885989666, "kl": 0.011021542362868786, "learning_rate": 4.9914675767918095e-06, "loss": 0.1403, "num_tokens": 83109920.0, "reward": 1.0500000715255737, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.7870595455169678, "sampling/importance_sampling_ratio/mean": 0.4383944571018219, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.027801752090454, "sampling/sampling_logp_difference/mean": 0.014046082273125648, "step": 784, "step_time": 72.12587340758182 }, { "clip_ratio/high_max": 0.011061026132665575, "clip_ratio/high_mean": 0.0031237077491823584, "clip_ratio/low_mean": 0.003735351587238256, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006859059329144657, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 974.0, "completions/mean_length": 946.1000366210938, "completions/mean_terminated_length": 677.7777709960938, "completions/min_length": 436.0, "completions/min_terminated_length": 436.0, "entropy": 0.1331197014078498, "epoch": 0.4465301478953356, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01656503602862358, "kl": 0.010683913598768413, "learning_rate": 4.986348122866894e-06, "loss": -0.0041, "num_tokens": 83215492.0, "reward": 0.42500001192092896, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.879626989364624, "sampling/importance_sampling_ratio/mean": 0.23111262917518616, "sampling/importance_sampling_ratio/min": 0.0012774321949109435, "sampling/sampling_logp_difference/max": 3.2991695404052734, "sampling/sampling_logp_difference/mean": 0.014233670197427273, "step": 785, "step_time": 72.93335833307356 }, { "clip_ratio/high_max": 0.016012826119549572, "clip_ratio/high_mean": 0.0049570879054954275, "clip_ratio/low_mean": 0.002514648425858468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007471736171282828, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 924.7000122070312, "completions/mean_terminated_length": 718.4615478515625, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.1648537814617157, "epoch": 0.447098976109215, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019929517060518265, "kl": 0.012995619093999267, "learning_rate": 4.9812286689419795e-06, "loss": 0.0435, "num_tokens": 83325304.0, "reward": 0.6000000238418579, "reward_std": 0.9001424908638, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 1.5951353311538696, "sampling/importance_sampling_ratio/mean": 0.262187659740448, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.553797960281372, "sampling/sampling_logp_difference/mean": 0.015977006405591965, "step": 786, "step_time": 74.46975604444742 }, { "clip_ratio/high_max": 0.014929377823136747, "clip_ratio/high_mean": 0.004955467738909647, "clip_ratio/low_mean": 0.0016320649829140166, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006587532814592123, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 739.9000244140625, "completions/mean_terminated_length": 632.137939453125, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.11085954122245312, "epoch": 0.44766780432309444, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015717746689915657, "kl": 0.02015065460000187, "learning_rate": 4.976109215017065e-06, "loss": 0.1122, "num_tokens": 83426980.0, "reward": 1.2000000476837158, "reward_std": 0.8533493876457214, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.637347936630249, "sampling/importance_sampling_ratio/mean": 0.2226032018661499, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7036757469177246, "sampling/sampling_logp_difference/mean": 0.01435531210154295, "step": 787, "step_time": 72.56578254583292 }, { "clip_ratio/high_max": 0.009061879885848612, "clip_ratio/high_mean": 0.0032837903854670003, "clip_ratio/low_mean": 0.0018881385694839992, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005171928874915466, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 885.0, "completions/mean_length": 866.125, "completions/mean_terminated_length": 652.5294189453125, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.08603838551789522, "epoch": 0.4482366325369738, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015513059683144093, "kl": 0.011253281438257545, "learning_rate": 4.97098976109215e-06, "loss": 0.0594, "num_tokens": 83528345.0, "reward": 0.7750000357627869, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.995875120162964, "sampling/importance_sampling_ratio/mean": 0.34621888399124146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0276317596435547, "sampling/sampling_logp_difference/mean": 0.01112087070941925, "step": 788, "step_time": 71.00228848611005 }, { "clip_ratio/high_max": 0.011583477316889912, "clip_ratio/high_mean": 0.00460096936149057, "clip_ratio/low_mean": 0.0021408172615338117, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0067417865502648056, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 797.3500366210938, "completions/mean_terminated_length": 570.7000122070312, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.09837185498327017, "epoch": 0.44880546075085326, "frac_reward_zero_std": 0.0, "grad_norm": 0.02365829050540924, "kl": 0.019514864310622215, "learning_rate": 4.965870307167235e-06, "loss": -0.0672, "num_tokens": 83632895.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.981230616569519, "sampling/importance_sampling_ratio/mean": 0.29159408807754517, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5876669883728027, "sampling/sampling_logp_difference/mean": 0.012926684692502022, "step": 789, "step_time": 73.35916100419126 }, { "clip_ratio/high_max": 0.009707346966024488, "clip_ratio/high_mean": 0.002365542208281113, "clip_ratio/low_mean": 0.0029296874490682967, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00529522990109399, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 962.0, "completions/mean_terminated_length": 748.4444580078125, "completions/min_length": 483.0, "completions/min_terminated_length": 483.0, "entropy": 0.1849030703306198, "epoch": 0.44937428896473264, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022402269765734673, "kl": 0.020864290883764625, "learning_rate": 4.960750853242321e-06, "loss": -0.0292, "num_tokens": 83748327.0, "reward": 0.4000000059604645, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290207743644714, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.9774391651153564, "sampling/importance_sampling_ratio/mean": 0.34153151512145996, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.809530735015869, "sampling/sampling_logp_difference/mean": 0.017561232671141624, "step": 790, "step_time": 78.62234668293968 }, { "clip_ratio/high_max": 0.008784434059634805, "clip_ratio/high_mean": 0.0019376097188796848, "clip_ratio/low_mean": 0.0005859375105501385, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002523547169403173, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 888.625, "completions/mean_terminated_length": 663.0000610351562, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "entropy": 0.0850020432844758, "epoch": 0.4499431171786121, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01889333687722683, "kl": 0.013014899042900652, "learning_rate": 4.955631399317407e-06, "loss": -0.0211, "num_tokens": 83860832.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.9700546264648438, "sampling/importance_sampling_ratio/mean": 0.46102532744407654, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.459902763366699, "sampling/sampling_logp_difference/mean": 0.01148380246013403, "step": 791, "step_time": 74.89261787733994 }, { "clip_ratio/high_max": 0.014814949710853398, "clip_ratio/high_mean": 0.004107272165128961, "clip_ratio/low_mean": 0.0017739988688845187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005881270859390497, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 868.4249877929688, "completions/mean_terminated_length": 609.1333618164062, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.09094440937042236, "epoch": 0.45051194539249145, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02644263207912445, "kl": 0.010391674237325788, "learning_rate": 4.950511945392492e-06, "loss": 0.0284, "num_tokens": 83968289.0, "reward": 0.699999988079071, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.091893196105957, "sampling/importance_sampling_ratio/mean": 0.449718177318573, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4622960090637207, "sampling/sampling_logp_difference/mean": 0.011932313442230225, "step": 792, "step_time": 74.05866423319094 }, { "clip_ratio/high_max": 0.014950979151763022, "clip_ratio/high_mean": 0.005581180623266846, "clip_ratio/low_mean": 0.0020740203326568007, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007655200839508325, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.0, "completions/mean_length": 720.6500244140625, "completions/mean_terminated_length": 538.6400146484375, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.13862550724297762, "epoch": 0.4510807736063709, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04725845903158188, "kl": 0.01583783864043653, "learning_rate": 4.945392491467577e-06, "loss": 0.0316, "num_tokens": 84064851.0, "reward": 1.100000023841858, "reward_std": 0.9554164409637451, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.553635835647583, "sampling/importance_sampling_ratio/mean": 0.445171058177948, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.974778890609741, "sampling/sampling_logp_difference/mean": 0.01636754535138607, "step": 793, "step_time": 68.82476858003065 }, { "clip_ratio/high_max": 0.009099540824536234, "clip_ratio/high_mean": 0.0023931163450470194, "clip_ratio/low_mean": 0.00129394532450533, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036870615876978263, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 906.0, "completions/mean_length": 912.75, "completions/mean_terminated_length": 529.5555419921875, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.1200149031355977, "epoch": 0.45164960182025027, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01426956057548523, "kl": 0.01216210686834529, "learning_rate": 4.940273037542662e-06, "loss": -0.0337, "num_tokens": 84174265.0, "reward": 0.45000001788139343, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290207743644714, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.45149302482605, "sampling/importance_sampling_ratio/mean": 0.3168027997016907, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8776884078979492, "sampling/sampling_logp_difference/mean": 0.013329214416444302, "step": 794, "step_time": 73.91743139969185 }, { "clip_ratio/high_max": 0.016454204742331058, "clip_ratio/high_mean": 0.0056815545103745535, "clip_ratio/low_mean": 0.0025037338673428167, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008185288344975561, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 854.2000122070312, "completions/mean_terminated_length": 666.5263061523438, "completions/min_length": 442.0, "completions/min_terminated_length": 442.0, "entropy": 0.12222269456833601, "epoch": 0.4522184300341297, "frac_reward_zero_std": 0.0, "grad_norm": 0.009019812569022179, "kl": 0.011473071528598666, "learning_rate": 4.9351535836177475e-06, "loss": 0.0564, "num_tokens": 84282977.0, "reward": 0.925000011920929, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.1451926231384277, "sampling/importance_sampling_ratio/mean": 0.21072843670845032, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.941199779510498, "sampling/sampling_logp_difference/mean": 0.014230954460799694, "step": 795, "step_time": 76.25721267377958 }, { "clip_ratio/high_max": 0.01325881463708356, "clip_ratio/high_mean": 0.00465805662679486, "clip_ratio/low_mean": 0.0015380859695142135, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006196142523549497, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 835.875, "completions/mean_terminated_length": 581.3529663085938, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.10543681122362614, "epoch": 0.4527872582480091, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024326929822564125, "kl": 0.012088545132428408, "learning_rate": 4.930034129692833e-06, "loss": 0.0574, "num_tokens": 84379100.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.7651267051696777, "sampling/importance_sampling_ratio/mean": 0.37894073128700256, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4490160942077637, "sampling/sampling_logp_difference/mean": 0.014180110767483711, "step": 796, "step_time": 72.46358499699272 }, { "clip_ratio/high_max": 0.013353908783756196, "clip_ratio/high_mean": 0.0033678754116408527, "clip_ratio/low_mean": 0.0016113281235448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00497920362977311, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 647.0, "completions/mean_length": 750.9000244140625, "completions/mean_terminated_length": 449.0526428222656, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.08063932321965694, "epoch": 0.4533560864618885, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023648152127861977, "kl": 0.012023694347590208, "learning_rate": 4.9249146757679176e-06, "loss": 0.0285, "num_tokens": 84489640.0, "reward": 0.9000000357627869, "reward_std": 0.9818872213363647, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.1327970027923584, "sampling/importance_sampling_ratio/mean": 0.4022600054740906, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8360776901245117, "sampling/sampling_logp_difference/mean": 0.011298633180558681, "step": 797, "step_time": 72.23204774502665 }, { "clip_ratio/high_max": 0.011301551130600274, "clip_ratio/high_mean": 0.002593240060377866, "clip_ratio/low_mean": 0.002270507815410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004863747832132503, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 989.25, "completions/mean_terminated_length": 825.4285888671875, "completions/min_length": 691.0, "completions/min_terminated_length": 691.0, "entropy": 0.1313528660684824, "epoch": 0.4539249146757679, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02526717260479927, "kl": 0.008115611562971026, "learning_rate": 4.919795221843004e-06, "loss": 0.1226, "num_tokens": 84605242.0, "reward": 0.32500001788139343, "reward_std": 0.7298577427864075, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.8692665100097656, "sampling/importance_sampling_ratio/mean": 0.3427511751651764, "sampling/importance_sampling_ratio/min": 0.0025147416163235903, "sampling/sampling_logp_difference/max": 1.3340288400650024, "sampling/sampling_logp_difference/mean": 0.01362984161823988, "step": 798, "step_time": 85.87669117376208 }, { "clip_ratio/high_max": 0.016023606644012034, "clip_ratio/high_mean": 0.004340823827078566, "clip_ratio/low_mean": 0.0013916015541326487, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005732425313908607, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 978.0, "completions/mean_length": 829.4750366210938, "completions/mean_terminated_length": 653.4761962890625, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 0.15327978134155273, "epoch": 0.45449374288964733, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.027668925002217293, "kl": 0.015307909110561013, "learning_rate": 4.914675767918089e-06, "loss": 0.0249, "num_tokens": 84713477.0, "reward": 1.024999976158142, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.5570132732391357, "sampling/importance_sampling_ratio/mean": 0.3718101382255554, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.71191668510437, "sampling/sampling_logp_difference/mean": 0.017324155196547508, "step": 799, "step_time": 74.9241526001133 }, { "clip_ratio/high_max": 0.003349609673023224, "clip_ratio/high_mean": 0.000669921952066943, "clip_ratio/low_mean": 0.001464843768189894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002134765702066943, "completions/clipped_ratio": 0.925000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 917.0, "completions/mean_length": 999.625, "completions/mean_terminated_length": 699.0, "completions/min_length": 528.0, "completions/min_terminated_length": 528.0, "entropy": 0.0950727891176939, "epoch": 0.4550625711035267, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.02099204994738102, "kl": 0.007714714505709708, "learning_rate": 4.909556313993175e-06, "loss": 0.0064, "num_tokens": 84824478.0, "reward": 0.15000000596046448, "reward_std": 0.5334935784339905, "rewards/format_reward/mean": 0.07500000298023224, "rewards/format_reward/std": 0.26674678921699524, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.494938850402832, "sampling/importance_sampling_ratio/mean": 0.5194323658943176, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4744656085968018, "sampling/sampling_logp_difference/mean": 0.009979038499295712, "step": 800, "step_time": 75.49126891954802 }, { "clip_ratio/high_max": 0.015251674456521869, "clip_ratio/high_mean": 0.00628614125889726, "clip_ratio/low_mean": 0.0022216796733118827, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.008507821010425687, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 821.1500244140625, "completions/mean_terminated_length": 596.9473876953125, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.10967954993247986, "epoch": 0.45563139931740615, "frac_reward_zero_std": 0.0, "grad_norm": 0.02855069935321808, "kl": 0.016031240113079548, "learning_rate": 4.904436860068259e-06, "loss": -0.1029, "num_tokens": 84917556.0, "reward": 0.949999988079071, "reward_std": 1.0114725828170776, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.7127788066864014, "sampling/importance_sampling_ratio/mean": 0.3636232018470764, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.8178956508636475, "sampling/sampling_logp_difference/mean": 0.013164222240447998, "step": 801, "step_time": 71.45596042182297 }, { "clip_ratio/high_max": 0.01420693943509832, "clip_ratio/high_mean": 0.0038383438804885373, "clip_ratio/low_mean": 0.0010498046867724042, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00488814854179509, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 869.7999877929688, "completions/mean_terminated_length": 549.5384521484375, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.07337130513042212, "epoch": 0.45620022753128553, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022767238318920135, "kl": 0.015448034042492509, "learning_rate": 4.899317406143345e-06, "loss": 0.0109, "num_tokens": 85024724.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.2584385871887207, "sampling/importance_sampling_ratio/mean": 0.44801443815231323, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6553125381469727, "sampling/sampling_logp_difference/mean": 0.010656344704329967, "step": 802, "step_time": 73.80305458698422 }, { "clip_ratio/high_max": 0.014069286873564124, "clip_ratio/high_mean": 0.003981451882282272, "clip_ratio/low_mean": 0.0013901376369176432, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005371589621063322, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 971.0, "completions/mean_length": 793.4249877929688, "completions/mean_terminated_length": 538.5789794921875, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.10071710497140884, "epoch": 0.45676905574516496, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.021916046738624573, "kl": 0.011429959442466497, "learning_rate": 4.89419795221843e-06, "loss": -0.0261, "num_tokens": 85122805.0, "reward": 0.75, "reward_std": 0.8697184920310974, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 1.9614942073822021, "sampling/importance_sampling_ratio/mean": 0.5367865562438965, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9634785652160645, "sampling/sampling_logp_difference/mean": 0.012720974162220955, "step": 803, "step_time": 71.29563872003928 }, { "clip_ratio/high_max": 0.011997272493317723, "clip_ratio/high_mean": 0.004026762835565023, "clip_ratio/low_mean": 0.0018798828023136593, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005906645703362301, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 902.7750244140625, "completions/mean_terminated_length": 651.0, "completions/min_length": 490.0, "completions/min_terminated_length": 490.0, "entropy": 0.06733466824516654, "epoch": 0.45733788395904434, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02072332054376602, "kl": 0.010529767023399472, "learning_rate": 4.889078498293516e-06, "loss": 0.0687, "num_tokens": 85229988.0, "reward": 0.5250000357627869, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.754275321960449, "sampling/importance_sampling_ratio/mean": 0.41134780645370483, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.184748888015747, "sampling/sampling_logp_difference/mean": 0.009889977052807808, "step": 804, "step_time": 73.2365841253195 }, { "clip_ratio/high_max": 0.01761269848793745, "clip_ratio/high_mean": 0.0061394226213451475, "clip_ratio/low_mean": 0.002246093710709829, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008385516470298171, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 778.4000244140625, "completions/mean_terminated_length": 646.1538696289062, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.18884060718119144, "epoch": 0.4579067121729238, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026762649416923523, "kl": 0.016517503419891, "learning_rate": 4.8839590443686e-06, "loss": 0.0107, "num_tokens": 85334476.0, "reward": 1.3000000715255737, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.1321396827697754, "sampling/importance_sampling_ratio/mean": 0.5194092988967896, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4341092109680176, "sampling/sampling_logp_difference/mean": 0.018297770991921425, "step": 805, "step_time": 73.07273914734833 }, { "clip_ratio/high_max": 0.014711404335685074, "clip_ratio/high_mean": 0.005368670143070631, "clip_ratio/low_mean": 0.0021484374738065526, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007517107878811657, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 867.7250366210938, "completions/mean_terminated_length": 695.0, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "entropy": 0.1320924162864685, "epoch": 0.45847554038680316, "frac_reward_zero_std": 0.0, "grad_norm": 0.021024907007813454, "kl": 0.012150665395893157, "learning_rate": 4.8788395904436865e-06, "loss": 0.0279, "num_tokens": 85444017.0, "reward": 0.8500000238418579, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.968319058418274, "sampling/importance_sampling_ratio/mean": 0.36010628938674927, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9273266792297363, "sampling/sampling_logp_difference/mean": 0.014476028271019459, "step": 806, "step_time": 74.37577601475641 }, { "clip_ratio/high_max": 0.010899424785748124, "clip_ratio/high_mean": 0.0028114497108617797, "clip_ratio/low_mean": 0.002026367175858468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004837816784856841, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 938.0, "completions/mean_length": 911.7750244140625, "completions/mean_terminated_length": 575.1000366210938, "completions/min_length": 406.0, "completions/min_terminated_length": 406.0, "entropy": 0.12990992702543736, "epoch": 0.4590443686006826, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0229233056306839, "kl": 0.011010932736098766, "learning_rate": 4.873720136518772e-06, "loss": 0.0334, "num_tokens": 85555976.0, "reward": 0.5, "reward_std": 0.8770580291748047, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.4328935146331787, "sampling/importance_sampling_ratio/mean": 0.36964064836502075, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1386423110961914, "sampling/sampling_logp_difference/mean": 0.013560662046074867, "step": 807, "step_time": 74.24077794817276 }, { "clip_ratio/high_max": 0.014585459954105318, "clip_ratio/high_mean": 0.006086766079533845, "clip_ratio/low_mean": 0.002150210209947545, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008236976456828415, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 735.6500244140625, "completions/mean_terminated_length": 580.3846435546875, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.15021373517811298, "epoch": 0.459613196814562, "frac_reward_zero_std": 0.0, "grad_norm": 0.03104395419359207, "kl": 0.016474913456477225, "learning_rate": 4.868600682593857e-06, "loss": -0.1404, "num_tokens": 85652218.0, "reward": 1.100000023841858, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.2843973636627197, "sampling/importance_sampling_ratio/mean": 0.5571491122245789, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9801018238067627, "sampling/sampling_logp_difference/mean": 0.015613081865012646, "step": 808, "step_time": 70.70946526504122 }, { "clip_ratio/high_max": 0.010780494078062475, "clip_ratio/high_mean": 0.00280175257648807, "clip_ratio/low_mean": 0.0007080078139551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003509760368615389, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 627.0250244140625, "completions/mean_terminated_length": 556.9705810546875, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.10175462160259485, "epoch": 0.4601820250284414, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.035604387521743774, "kl": 0.017341691767796874, "learning_rate": 4.863481228668942e-06, "loss": 0.0096, "num_tokens": 85746787.0, "reward": 1.7000000476837158, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.8500000238418579, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.8500000238418579, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.4431798458099365, "sampling/importance_sampling_ratio/mean": 0.5750686526298523, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2713162899017334, "sampling/sampling_logp_difference/mean": 0.014514319598674774, "step": 809, "step_time": 68.26500780391507 }, { "clip_ratio/high_max": 0.0117234144709073, "clip_ratio/high_mean": 0.004441284880158491, "clip_ratio/low_mean": 0.0019775390610448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006418823963031173, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 794.125, "completions/mean_terminated_length": 606.0454711914062, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.12195855006575584, "epoch": 0.46075085324232085, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029750626534223557, "kl": 0.015067498083226383, "learning_rate": 4.858361774744027e-06, "loss": -0.0146, "num_tokens": 85856768.0, "reward": 1.0, "reward_std": 0.9607689380645752, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.764572858810425, "sampling/importance_sampling_ratio/mean": 0.46317654848098755, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4624810218811035, "sampling/sampling_logp_difference/mean": 0.014417153783142567, "step": 810, "step_time": 75.86759297340177 }, { "clip_ratio/high_max": 0.015982180717401206, "clip_ratio/high_mean": 0.005832304210343864, "clip_ratio/low_mean": 0.0033450137998443097, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009177318017464131, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 846.8250122070312, "completions/mean_terminated_length": 607.11767578125, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.17502127029001713, "epoch": 0.4613196814562002, "frac_reward_zero_std": 0.0, "grad_norm": 0.02391701564192772, "kl": 0.015487799770198762, "learning_rate": 4.853242320819113e-06, "loss": 0.0998, "num_tokens": 85970657.0, "reward": 0.7750000357627869, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.2253010272979736, "sampling/importance_sampling_ratio/mean": 0.4549678862094879, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.636091709136963, "sampling/sampling_logp_difference/mean": 0.017346343025565147, "step": 811, "step_time": 77.098138875328 }, { "clip_ratio/high_max": 0.014946873707231134, "clip_ratio/high_mean": 0.005452329729450867, "clip_ratio/low_mean": 0.0030517578270519152, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.008504087571054697, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 858.5250244140625, "completions/mean_terminated_length": 634.6470336914062, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.14066811371594667, "epoch": 0.46188850967007966, "frac_reward_zero_std": 0.0, "grad_norm": 0.04245057329535484, "kl": 0.01544225588440895, "learning_rate": 4.848122866894197e-06, "loss": -0.1456, "num_tokens": 86079086.0, "reward": 0.675000011920929, "reward_std": 0.8589648008346558, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.8408212661743164, "sampling/importance_sampling_ratio/mean": 0.4163495600223541, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.466766834259033, "sampling/sampling_logp_difference/mean": 0.015515071339905262, "step": 812, "step_time": 75.16549829253927 }, { "clip_ratio/high_max": 0.013417607042356394, "clip_ratio/high_mean": 0.005196617919864366, "clip_ratio/low_mean": 0.0019552919584384654, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007151910045649856, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 763.8500366210938, "completions/mean_terminated_length": 623.7692260742188, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.125243223272264, "epoch": 0.46245733788395904, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021026644855737686, "kl": 0.018913391511887312, "learning_rate": 4.843003412969283e-06, "loss": -0.0385, "num_tokens": 86180360.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.737311840057373, "sampling/importance_sampling_ratio/mean": 0.4139847755432129, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3139631748199463, "sampling/sampling_logp_difference/mean": 0.015963584184646606, "step": 813, "step_time": 69.80777061870322 }, { "clip_ratio/high_max": 0.011695406050421298, "clip_ratio/high_mean": 0.0028734346851706505, "clip_ratio/low_mean": 0.001367187505820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004240622220095247, "completions/clipped_ratio": 0.17499999701976776, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 686.875, "completions/mean_terminated_length": 615.3636474609375, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.16377700306475163, "epoch": 0.4630261660978385, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.013586693443357944, "kl": 0.018589492654427886, "learning_rate": 4.837883959044369e-06, "loss": -0.0415, "num_tokens": 86288891.0, "reward": 1.649999976158142, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.824999988079071, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.824999988079071, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.2583985328674316, "sampling/importance_sampling_ratio/mean": 0.401745468378067, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.880742073059082, "sampling/sampling_logp_difference/mean": 0.019394805654883385, "step": 814, "step_time": 72.40316414623521 }, { "clip_ratio/high_max": 0.014822986093349755, "clip_ratio/high_mean": 0.005621900985715911, "clip_ratio/low_mean": 0.0026380631170468405, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.008259964408352971, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 723.4249877929688, "completions/mean_terminated_length": 501.2608642578125, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.13090565335005522, "epoch": 0.46359499431171786, "frac_reward_zero_std": 0.0, "grad_norm": 0.029852427542209625, "kl": 0.016720004961825907, "learning_rate": 4.8327645051194545e-06, "loss": 0.1122, "num_tokens": 86395900.0, "reward": 0.949999988079071, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.272554874420166, "sampling/importance_sampling_ratio/mean": 0.36817869544029236, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9630546569824219, "sampling/sampling_logp_difference/mean": 0.015171191655099392, "step": 815, "step_time": 73.68067609495483 }, { "clip_ratio/high_max": 0.011890676454640925, "clip_ratio/high_mean": 0.004017467588710133, "clip_ratio/low_mean": 0.0015395571626868332, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005557024705922231, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 903.0, "completions/mean_length": 872.6749877929688, "completions/mean_terminated_length": 667.941162109375, "completions/min_length": 460.0, "completions/min_terminated_length": 460.0, "entropy": 0.10186602361500263, "epoch": 0.4641638225255973, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01886589266359806, "kl": 0.010729310452006757, "learning_rate": 4.827645051194539e-06, "loss": -0.0108, "num_tokens": 86498559.0, "reward": 0.75, "reward_std": 0.9268086552619934, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.658156156539917, "sampling/importance_sampling_ratio/mean": 0.5660079717636108, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2185425758361816, "sampling/sampling_logp_difference/mean": 0.01221980806440115, "step": 816, "step_time": 72.08325650775805 }, { "clip_ratio/high_max": 0.01126414950704202, "clip_ratio/high_mean": 0.003508256340865046, "clip_ratio/low_mean": 0.003759765619179234, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.007268022105563432, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 938.0, "completions/mean_terminated_length": 711.2727661132812, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.1336608249694109, "epoch": 0.46473265073947667, "frac_reward_zero_std": 0.0, "grad_norm": 0.04941864684224129, "kl": 0.012994479387998581, "learning_rate": 4.8225255972696246e-06, "loss": 0.0469, "num_tokens": 86613183.0, "reward": 0.45000001788139343, "reward_std": 0.7828285098075867, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.8496406078338623, "sampling/importance_sampling_ratio/mean": 0.5078875422477722, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1277880668640137, "sampling/sampling_logp_difference/mean": 0.014328034594655037, "step": 817, "step_time": 76.03205136628821 }, { "clip_ratio/high_max": 0.01168000977486372, "clip_ratio/high_mean": 0.0029623142545460723, "clip_ratio/low_mean": 0.0008544921711290954, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038168063474586233, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 789.4000244140625, "completions/mean_terminated_length": 472.0, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.11995908059179783, "epoch": 0.4653014789533561, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.007180377840995789, "kl": 0.01225535717094317, "learning_rate": 4.81740614334471e-06, "loss": -0.0118, "num_tokens": 86714223.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.262326717376709, "sampling/importance_sampling_ratio/mean": 0.4554556906223297, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.071913242340088, "sampling/sampling_logp_difference/mean": 0.014472290873527527, "step": 818, "step_time": 72.59099878696725 }, { "clip_ratio/high_max": 0.015085455670487136, "clip_ratio/high_mean": 0.004495135901379399, "clip_ratio/low_mean": 0.0010009765574068297, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005496112426044419, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 962.0, "completions/mean_length": 782.1500244140625, "completions/mean_terminated_length": 603.3912963867188, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "entropy": 0.10991091001778841, "epoch": 0.4658703071672355, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015651656314730644, "kl": 0.016966343857347965, "learning_rate": 4.812286689419795e-06, "loss": -0.0388, "num_tokens": 86828253.0, "reward": 1.149999976158142, "reward_std": 1.0012813806533813, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.2688003778457642, "sampling/importance_sampling_ratio/mean": 0.3029162287712097, "sampling/importance_sampling_ratio/min": 0.0009558509918861091, "sampling/sampling_logp_difference/max": 2.7983760833740234, "sampling/sampling_logp_difference/mean": 0.014255058951675892, "step": 819, "step_time": 77.16410172404721 }, { "clip_ratio/high_max": 0.017439099960029125, "clip_ratio/high_mean": 0.005320079071680084, "clip_ratio/low_mean": 0.0019287109480501385, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007248790032463148, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 870.1749877929688, "completions/mean_terminated_length": 639.4375, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.1181449769064784, "epoch": 0.4664391353811149, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027380408719182014, "kl": 0.016007276950404048, "learning_rate": 4.80716723549488e-06, "loss": 0.1036, "num_tokens": 86933148.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.0382041931152344, "sampling/importance_sampling_ratio/mean": 0.24549052119255066, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5404727458953857, "sampling/sampling_logp_difference/mean": 0.013600355945527554, "step": 820, "step_time": 73.73046368197538 }, { "clip_ratio/high_max": 0.005991287645883858, "clip_ratio/high_mean": 0.0011982575233560055, "clip_ratio/low_mean": 0.0018310547166038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030293122108560055, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 871.0, "completions/mean_length": 900.0750122070312, "completions/mean_terminated_length": 573.3636474609375, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.09646722488105297, "epoch": 0.4670079635949943, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.012649224139750004, "kl": 0.010492983157746494, "learning_rate": 4.8020477815699654e-06, "loss": 0.0433, "num_tokens": 87040335.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.9510433673858643, "sampling/importance_sampling_ratio/mean": 0.3916040062904358, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1965959072113037, "sampling/sampling_logp_difference/mean": 0.011627983301877975, "step": 821, "step_time": 72.63578336220235 }, { "clip_ratio/high_max": 0.012982785934582353, "clip_ratio/high_mean": 0.004547713193460368, "clip_ratio/low_mean": 0.0025634765115682967, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007111189770512283, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 855.0, "completions/mean_length": 896.7750244140625, "completions/mean_terminated_length": 705.9375, "completions/min_length": 470.0, "completions/min_terminated_length": 470.0, "entropy": 0.11234031151980162, "epoch": 0.46757679180887374, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.06979728490114212, "kl": 0.010983921121805906, "learning_rate": 4.796928327645052e-06, "loss": -0.0804, "num_tokens": 87138166.0, "reward": 0.699999988079071, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.9479618072509766, "sampling/importance_sampling_ratio/mean": 0.42188185453414917, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.053309917449951, "sampling/sampling_logp_difference/mean": 0.013642646372318268, "step": 822, "step_time": 71.48782822815701 }, { "clip_ratio/high_max": 0.014363649301230907, "clip_ratio/high_mean": 0.0044680687569780275, "clip_ratio/low_mean": 0.0023925781351863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006860646884888411, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 849.1500244140625, "completions/mean_terminated_length": 612.5882568359375, "completions/min_length": 417.0, "completions/min_terminated_length": 417.0, "entropy": 0.10680371895432472, "epoch": 0.4681456200227531, "frac_reward_zero_std": 0.0, "grad_norm": 0.024225488305091858, "kl": 0.012578579713590443, "learning_rate": 4.791808873720137e-06, "loss": 0.0318, "num_tokens": 87245348.0, "reward": 0.7750000357627869, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.0081095695495605, "sampling/importance_sampling_ratio/mean": 0.27878537774086, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0734634399414062, "sampling/sampling_logp_difference/mean": 0.01258369442075491, "step": 823, "step_time": 75.63963141106069 }, { "clip_ratio/high_max": 0.007511907868320122, "clip_ratio/high_mean": 0.002346847628359683, "clip_ratio/low_mean": 0.0019287109753349796, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004275558574590832, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 897.4500122070312, "completions/mean_terminated_length": 662.4285888671875, "completions/min_length": 412.0, "completions/min_terminated_length": 412.0, "entropy": 0.06931722396984696, "epoch": 0.46871444823663255, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02509995736181736, "kl": 0.008268278383184224, "learning_rate": 4.786689419795222e-06, "loss": 0.0407, "num_tokens": 87339526.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.8788129091262817, "sampling/importance_sampling_ratio/mean": 0.4477773606777191, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.835970401763916, "sampling/sampling_logp_difference/mean": 0.0092953871935606, "step": 824, "step_time": 73.07310837926343 }, { "clip_ratio/high_max": 0.015732234460301697, "clip_ratio/high_mean": 0.005164727059309371, "clip_ratio/low_mean": 0.002827800824888982, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007992527971509844, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 914.0, "completions/mean_length": 808.2000122070312, "completions/mean_terminated_length": 569.6842041015625, "completions/min_length": 392.0, "completions/min_terminated_length": 392.0, "entropy": 0.12350642774254084, "epoch": 0.46928327645051193, "frac_reward_zero_std": 0.0, "grad_norm": 0.03260355070233345, "kl": 0.012972105876542628, "learning_rate": 4.781569965870307e-06, "loss": -0.0557, "num_tokens": 87450950.0, "reward": 0.9000000357627869, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.460913896560669, "sampling/importance_sampling_ratio/mean": 0.46855616569519043, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2653262615203857, "sampling/sampling_logp_difference/mean": 0.01437936257570982, "step": 825, "step_time": 73.94458099710755 }, { "clip_ratio/high_max": 0.00821637047920376, "clip_ratio/high_mean": 0.0020732418342959136, "clip_ratio/low_mean": 0.0019506858552631456, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004023927642265335, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 850.0, "completions/mean_length": 885.7000122070312, "completions/mean_terminated_length": 563.0, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.08847972005605698, "epoch": 0.46985210466439137, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03421976789832115, "kl": 0.011214570724405348, "learning_rate": 4.776450511945393e-06, "loss": -0.0746, "num_tokens": 87559346.0, "reward": 0.5, "reward_std": 0.8164966106414795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.8883216381073, "sampling/importance_sampling_ratio/mean": 0.4260641038417816, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8837794065475464, "sampling/sampling_logp_difference/mean": 0.011270582675933838, "step": 826, "step_time": 75.26025950303301 }, { "clip_ratio/high_max": 0.014085284783504903, "clip_ratio/high_mean": 0.004871367535088211, "clip_ratio/low_mean": 0.0016845702994032763, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006555937812663615, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 977.0, "completions/mean_length": 762.0, "completions/mean_terminated_length": 500.0, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.07100819237530231, "epoch": 0.47042093287827075, "frac_reward_zero_std": 0.0, "grad_norm": 0.018395831808447838, "kl": 0.016194722382351756, "learning_rate": 4.771331058020478e-06, "loss": 0.0185, "num_tokens": 87667938.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.6969916820526123, "sampling/importance_sampling_ratio/mean": 0.4044116139411926, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3351237773895264, "sampling/sampling_logp_difference/mean": 0.010905633680522442, "step": 827, "step_time": 73.56115618068725 }, { "clip_ratio/high_max": 0.0169764026068151, "clip_ratio/high_mean": 0.0062106940895318985, "clip_ratio/low_mean": 0.0016601562601863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007870850211475044, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 653.4249877929688, "completions/mean_terminated_length": 494.607177734375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.12604963220655918, "epoch": 0.4709897610921502, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013643737882375717, "kl": 0.01655187166761607, "learning_rate": 4.766211604095563e-06, "loss": -0.0108, "num_tokens": 87765003.0, "reward": 1.375, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.548758864402771, "sampling/importance_sampling_ratio/mean": 0.30457624793052673, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5947179794311523, "sampling/sampling_logp_difference/mean": 0.016194278374314308, "step": 828, "step_time": 70.18154449691065 }, { "clip_ratio/high_max": 0.01624340598937124, "clip_ratio/high_mean": 0.005552226764848456, "clip_ratio/low_mean": 0.0021484374956344254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007700664224103093, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 827.8250122070312, "completions/mean_terminated_length": 562.4117431640625, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.10731182899326086, "epoch": 0.47155858930602956, "frac_reward_zero_std": 0.0, "grad_norm": 0.020033162087202072, "kl": 0.015056084957905114, "learning_rate": 4.761092150170648e-06, "loss": 0.0135, "num_tokens": 87863244.0, "reward": 0.7250000238418579, "reward_std": 0.905467689037323, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.039334774017334, "sampling/importance_sampling_ratio/mean": 0.32512450218200684, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.79217529296875, "sampling/sampling_logp_difference/mean": 0.01329881977289915, "step": 829, "step_time": 73.1487100739032 }, { "clip_ratio/high_max": 0.01486045285128057, "clip_ratio/high_mean": 0.0039088891935534775, "clip_ratio/low_mean": 0.002465820318320766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006374709482770413, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 898.8250122070312, "completions/mean_terminated_length": 606.75, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.13792218454182148, "epoch": 0.472127417519909, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02098562940955162, "kl": 0.010458980628754944, "learning_rate": 4.755972696245734e-06, "loss": 0.0912, "num_tokens": 87976669.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.0052757263183594, "sampling/importance_sampling_ratio/mean": 0.4424569010734558, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4883766174316406, "sampling/sampling_logp_difference/mean": 0.013684185221791267, "step": 830, "step_time": 75.37030148762278 }, { "clip_ratio/high_max": 0.010310822923202068, "clip_ratio/high_mean": 0.0024010800407268107, "clip_ratio/low_mean": 0.0006183117911859881, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030193918355507776, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 595.0, "completions/mean_length": 750.75, "completions/mean_terminated_length": 448.7368469238281, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.10923328809440136, "epoch": 0.4726962457337884, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.03373700752854347, "kl": 0.015598608646541834, "learning_rate": 4.75085324232082e-06, "loss": 0.0134, "num_tokens": 88081659.0, "reward": 0.9000000357627869, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9463696479797363, "sampling/importance_sampling_ratio/mean": 0.44301480054855347, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4314775466918945, "sampling/sampling_logp_difference/mean": 0.013962347991764545, "step": 831, "step_time": 70.88577797077596 }, { "clip_ratio/high_max": 0.014139015693217516, "clip_ratio/high_mean": 0.004118397060665302, "clip_ratio/low_mean": 0.0019349909998709336, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006053388002328575, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 855.625, "completions/mean_terminated_length": 575.0, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.15250828117132187, "epoch": 0.4732650739476678, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019074110314249992, "kl": 0.014741162303835154, "learning_rate": 4.745733788395904e-06, "loss": 0.0172, "num_tokens": 88185964.0, "reward": 0.675000011920929, "reward_std": 0.9442809820175171, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.5230244398117065, "sampling/importance_sampling_ratio/mean": 0.30208316445350647, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7038114070892334, "sampling/sampling_logp_difference/mean": 0.016545798629522324, "step": 832, "step_time": 73.79396822373383 }, { "clip_ratio/high_max": 0.006747518316842616, "clip_ratio/high_mean": 0.0016542966186534613, "clip_ratio/low_mean": 0.0008789062521827873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0025332028890261427, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 960.0, "completions/mean_length": 756.6749877929688, "completions/mean_terminated_length": 559.0869750976562, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.0820742454379797, "epoch": 0.4738339021615472, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.018898596987128258, "kl": 0.013367490551900119, "learning_rate": 4.74061433447099e-06, "loss": 0.0024, "num_tokens": 88295543.0, "reward": 0.824999988079071, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.9597885608673096, "sampling/importance_sampling_ratio/mean": 0.4734590947628021, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4806759357452393, "sampling/sampling_logp_difference/mean": 0.011954958550632, "step": 833, "step_time": 71.5589750064537 }, { "clip_ratio/high_max": 0.017544369562529027, "clip_ratio/high_mean": 0.005248274886980653, "clip_ratio/low_mean": 0.0019322266089147888, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007180501474067569, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 929.0, "completions/mean_length": 859.8500366210938, "completions/mean_terminated_length": 586.2667236328125, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.14264238253235817, "epoch": 0.47440273037542663, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015099040232598782, "kl": 0.01559829618781805, "learning_rate": 4.735494880546075e-06, "loss": -0.0206, "num_tokens": 88403289.0, "reward": 0.699999988079071, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.1110353469848633, "sampling/importance_sampling_ratio/mean": 0.26557061076164246, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4432106018066406, "sampling/sampling_logp_difference/mean": 0.01526098232716322, "step": 834, "step_time": 73.43493877211586 }, { "clip_ratio/high_max": 0.010289539524819702, "clip_ratio/high_mean": 0.003637386893387884, "clip_ratio/low_mean": 0.0009033203023136593, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004540707275737077, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 829.0, "completions/mean_length": 856.1500244140625, "completions/mean_terminated_length": 604.375, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.12751500960439444, "epoch": 0.474971558589306, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014231855981051922, "kl": 0.013055508024990559, "learning_rate": 4.730375426621161e-06, "loss": -0.0375, "num_tokens": 88514135.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.6216228008270264, "sampling/importance_sampling_ratio/mean": 0.34735941886901855, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1475636959075928, "sampling/sampling_logp_difference/mean": 0.014131923206150532, "step": 835, "step_time": 74.86176950717345 }, { "clip_ratio/high_max": 0.014387746807187796, "clip_ratio/high_mean": 0.005224887645454146, "clip_ratio/low_mean": 0.0013971002463222248, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0066219878499396145, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 929.0, "completions/mean_length": 793.25, "completions/mean_terminated_length": 511.22222900390625, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.09583156928420067, "epoch": 0.47554038680318544, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.031584713608026505, "kl": 0.014433518517762423, "learning_rate": 4.725255972696245e-06, "loss": -0.0393, "num_tokens": 88617097.0, "reward": 0.8500000238418579, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.747159242630005, "sampling/importance_sampling_ratio/mean": 0.4635276794433594, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5543723106384277, "sampling/sampling_logp_difference/mean": 0.012078299187123775, "step": 836, "step_time": 72.19670917908661 }, { "clip_ratio/high_max": 0.013333877548575401, "clip_ratio/high_mean": 0.005022102821385488, "clip_ratio/low_mean": 0.002807617165672127, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.007829720008885488, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 877.4500122070312, "completions/mean_terminated_length": 633.2000122070312, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.12509040255099535, "epoch": 0.4761092150170648, "frac_reward_zero_std": 0.0, "grad_norm": 0.030725782737135887, "kl": 0.01132043485995382, "learning_rate": 4.7201365187713315e-06, "loss": 0.0048, "num_tokens": 88719659.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.518312931060791, "sampling/importance_sampling_ratio/mean": 0.418694406747818, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.473116159439087, "sampling/sampling_logp_difference/mean": 0.013911447487771511, "step": 837, "step_time": 73.03297881502658 }, { "clip_ratio/high_max": 0.012719982129056007, "clip_ratio/high_mean": 0.004314446559874341, "clip_ratio/low_mean": 0.001782226565410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0060966731980443, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 969.0, "completions/mean_length": 841.0, "completions/mean_terminated_length": 566.5, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.09048865176737309, "epoch": 0.47667804323094426, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.05020546168088913, "kl": 0.031171628856100142, "learning_rate": 4.715017064846417e-06, "loss": -0.0281, "num_tokens": 88822563.0, "reward": 0.625, "reward_std": 0.8378084897994995, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.726091146469116, "sampling/importance_sampling_ratio/mean": 0.32951751351356506, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8797061443328857, "sampling/sampling_logp_difference/mean": 0.012300875037908554, "step": 838, "step_time": 78.14136520586908 }, { "clip_ratio/high_max": 0.009709962119814008, "clip_ratio/high_mean": 0.002949856221675873, "clip_ratio/low_mean": 0.002246093761641532, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005195949896005914, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 940.1749877929688, "completions/mean_terminated_length": 826.7647094726562, "completions/min_length": 448.0, "completions/min_terminated_length": 448.0, "entropy": 0.11996039934456348, "epoch": 0.47724687144482364, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023418664932250977, "kl": 0.00943359814118594, "learning_rate": 4.709897610921502e-06, "loss": -0.0984, "num_tokens": 88933498.0, "reward": 0.6500000357627869, "reward_std": 0.8335896730422974, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 1.9155778884887695, "sampling/importance_sampling_ratio/mean": 0.28315114974975586, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.179722547531128, "sampling/sampling_logp_difference/mean": 0.013167700730264187, "step": 839, "step_time": 75.28779433201998 }, { "clip_ratio/high_max": 0.009924275625962764, "clip_ratio/high_mean": 0.003212291223462671, "clip_ratio/low_mean": 0.0008989715515781427, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004111262765945867, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 776.0, "completions/mean_length": 843.6000366210938, "completions/mean_terminated_length": 573.0, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.099806590937078, "epoch": 0.4778156996587031, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.029991164803504944, "kl": 0.016614333144389093, "learning_rate": 4.704778156996587e-06, "loss": 0.0466, "num_tokens": 89024642.0, "reward": 0.7250000238418579, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.540501117706299, "sampling/importance_sampling_ratio/mean": 0.6219164133071899, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.634368419647217, "sampling/sampling_logp_difference/mean": 0.012127647176384926, "step": 840, "step_time": 68.69855265389197 }, { "clip_ratio/high_max": 0.012485379469580948, "clip_ratio/high_mean": 0.003673189756227657, "clip_ratio/low_mean": 0.0019609616065281443, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0056341514573432505, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 979.0, "completions/mean_length": 821.1000366210938, "completions/mean_terminated_length": 637.5238037109375, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.17037764936685562, "epoch": 0.47838452787258245, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03855098783969879, "kl": 0.012098885257728398, "learning_rate": 4.6996587030716724e-06, "loss": 0.0244, "num_tokens": 89133678.0, "reward": 0.949999988079071, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.9516401290893555, "sampling/importance_sampling_ratio/mean": 0.6394914388656616, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7393304109573364, "sampling/sampling_logp_difference/mean": 0.016732454299926758, "step": 841, "step_time": 74.73919591819867 }, { "clip_ratio/high_max": 0.005848120170412585, "clip_ratio/high_mean": 0.001651530863455264, "clip_ratio/low_mean": 0.0011995948807452805, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002851125755114481, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 740.4249877929688, "completions/mean_terminated_length": 618.8928833007812, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.08339406875893474, "epoch": 0.4789533560864619, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.030317721888422966, "kl": 0.011195093509741127, "learning_rate": 4.694539249146758e-06, "loss": 0.1712, "num_tokens": 89232519.0, "reward": 1.3250000476837158, "reward_std": 0.9167249798774719, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.4703149795532227, "sampling/importance_sampling_ratio/mean": 0.48950856924057007, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.5937724113464355, "sampling/sampling_logp_difference/mean": 0.011650440283119678, "step": 842, "step_time": 70.79259292199276 }, { "clip_ratio/high_max": 0.013227464514784515, "clip_ratio/high_mean": 0.005665565782692283, "clip_ratio/low_mean": 0.0028494297002907842, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008514995541190729, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 721.5, "completions/mean_terminated_length": 540.0, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.12781080044806004, "epoch": 0.47952218430034127, "frac_reward_zero_std": 0.0, "grad_norm": 0.04841279238462448, "kl": 0.014952429221011698, "learning_rate": 4.689419795221843e-06, "loss": -0.1248, "num_tokens": 89333043.0, "reward": 0.9750000238418579, "reward_std": 0.8316649794578552, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.7520060539245605, "sampling/importance_sampling_ratio/mean": 0.47633591294288635, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.206057548522949, "sampling/sampling_logp_difference/mean": 0.014924317598342896, "step": 843, "step_time": 71.8850176979322 }, { "clip_ratio/high_max": 0.01405611849622801, "clip_ratio/high_mean": 0.004553794897219632, "clip_ratio/low_mean": 0.002368164077779511, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006921959051396698, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 859.0, "completions/mean_length": 882.75, "completions/mean_terminated_length": 647.3333740234375, "completions/min_length": 393.0, "completions/min_terminated_length": 393.0, "entropy": 0.1509931068867445, "epoch": 0.4800910125142207, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027265943586826324, "kl": 0.010088594863191247, "learning_rate": 4.684300341296928e-06, "loss": 0.0682, "num_tokens": 89438185.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.996174693107605, "sampling/importance_sampling_ratio/mean": 0.37227216362953186, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6523029804229736, "sampling/sampling_logp_difference/mean": 0.014668094925582409, "step": 844, "step_time": 74.9949199850671 }, { "clip_ratio/high_max": 0.01797419146168977, "clip_ratio/high_mean": 0.006838897796114907, "clip_ratio/low_mean": 0.00141601563700533, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008254913438577205, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 725.7999877929688, "completions/mean_terminated_length": 598.0, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.1406576642766595, "epoch": 0.48065984072810014, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024622365832328796, "kl": 0.02149946941062808, "learning_rate": 4.679180887372014e-06, "loss": -0.0773, "num_tokens": 89551401.0, "reward": 1.350000023841858, "reward_std": 0.9212588667869568, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.9692414999008179, "sampling/importance_sampling_ratio/mean": 0.38195621967315674, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.458726406097412, "sampling/sampling_logp_difference/mean": 0.01744532771408558, "step": 845, "step_time": 74.88447068003006 }, { "clip_ratio/high_max": 0.00447294776677154, "clip_ratio/high_mean": 0.0009455579565837979, "clip_ratio/low_mean": 0.0008300781246362021, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017756360757630318, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 761.2999877929688, "completions/mean_terminated_length": 546.3636474609375, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.11302698124200106, "epoch": 0.4812286689419795, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.02668043039739132, "kl": 0.011857790756039321, "learning_rate": 4.6740614334471e-06, "loss": 0.0008, "num_tokens": 89656861.0, "reward": 1.0500000715255737, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.541090250015259, "sampling/importance_sampling_ratio/mean": 0.40539684891700745, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9470279216766357, "sampling/sampling_logp_difference/mean": 0.01479263324290514, "step": 846, "step_time": 71.2952786711976 }, { "clip_ratio/high_max": 0.01063501468161121, "clip_ratio/high_mean": 0.0027958473365288228, "clip_ratio/low_mean": 0.002343749991268851, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0051395972550380975, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 819.5250244140625, "completions/mean_terminated_length": 569.6111450195312, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.10247681383043528, "epoch": 0.48179749715585896, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.012732179835438728, "kl": 0.012524482212029397, "learning_rate": 4.668941979522184e-06, "loss": -0.0197, "num_tokens": 89769554.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.6381278038024902, "sampling/importance_sampling_ratio/mean": 0.34993019700050354, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4655303955078125, "sampling/sampling_logp_difference/mean": 0.013148811645805836, "step": 847, "step_time": 74.34626027219929 }, { "clip_ratio/high_max": 0.017263651825487614, "clip_ratio/high_mean": 0.0049950303582591005, "clip_ratio/low_mean": 0.0011172754529980011, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006112305854912847, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 637.9750366210938, "completions/mean_terminated_length": 541.46875, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.11104667000472546, "epoch": 0.48236632536973834, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016265325248241425, "kl": 0.013354779221117496, "learning_rate": 4.66382252559727e-06, "loss": -0.0479, "num_tokens": 89869737.0, "reward": 1.5750000476837158, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.7749999761581421, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.9362106323242188, "sampling/importance_sampling_ratio/mean": 0.4802107810974121, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.265221357345581, "sampling/sampling_logp_difference/mean": 0.014859328046441078, "step": 848, "step_time": 67.94567494280636 }, { "clip_ratio/high_max": 0.01103460683953017, "clip_ratio/high_mean": 0.002349941903958097, "clip_ratio/low_mean": 0.001782226583600277, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004132168513024226, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 890.6500244140625, "completions/mean_terminated_length": 781.5454711914062, "completions/min_length": 421.0, "completions/min_terminated_length": 421.0, "entropy": 0.16262869723141193, "epoch": 0.48293515358361777, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009932911023497581, "kl": 0.011565706110559404, "learning_rate": 4.658703071672355e-06, "loss": 0.0069, "num_tokens": 89980083.0, "reward": 1.024999976158142, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.6167538166046143, "sampling/importance_sampling_ratio/mean": 0.3288201689720154, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4949672222137451, "sampling/sampling_logp_difference/mean": 0.01630004122853279, "step": 849, "step_time": 74.40131645509973 }, { "clip_ratio/high_max": 0.01542418054305017, "clip_ratio/high_mean": 0.004740882257465273, "clip_ratio/low_mean": 0.0012018277720926562, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005942710151430219, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 943.0, "completions/mean_length": 806.6749877929688, "completions/mean_terminated_length": 444.4666748046875, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.11251943930983543, "epoch": 0.48350398179749715, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.028249213472008705, "kl": 0.016163377673365176, "learning_rate": 4.6535836177474405e-06, "loss": 0.1076, "num_tokens": 90080310.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.9946954250335693, "sampling/importance_sampling_ratio/mean": 0.4988366961479187, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0796103477478027, "sampling/sampling_logp_difference/mean": 0.013766700401902199, "step": 850, "step_time": 72.65710858954117 }, { "clip_ratio/high_max": 0.015443007345311344, "clip_ratio/high_mean": 0.00730880000628531, "clip_ratio/low_mean": 0.0009765625109139364, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008285362448077649, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 867.0, "completions/mean_length": 737.3500366210938, "completions/mean_terminated_length": 599.3333129882812, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.09255257528275251, "epoch": 0.4840728100113766, "frac_reward_zero_std": 0.0, "grad_norm": 0.02800212800502777, "kl": 0.009985740121919662, "learning_rate": 4.648464163822525e-06, "loss": -0.0743, "num_tokens": 90185364.0, "reward": 1.350000023841858, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.834683656692505, "sampling/importance_sampling_ratio/mean": 0.5539084672927856, "sampling/importance_sampling_ratio/min": 0.006245004013180733, "sampling/sampling_logp_difference/max": 1.8753808736801147, "sampling/sampling_logp_difference/mean": 0.01237015426158905, "step": 851, "step_time": 72.26818371587433 }, { "clip_ratio/high_max": 0.015585547138471156, "clip_ratio/high_mean": 0.005937934212852269, "clip_ratio/low_mean": 0.002709960928768851, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.008647895127069205, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 930.0, "completions/mean_length": 808.1749877929688, "completions/mean_terminated_length": 484.4375, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.09766911994665861, "epoch": 0.48464163822525597, "frac_reward_zero_std": 0.0, "grad_norm": 0.05834142863750458, "kl": 0.013205848459620029, "learning_rate": 4.6433447098976105e-06, "loss": -0.0368, "num_tokens": 90293331.0, "reward": 0.7250000238418579, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.1388044357299805, "sampling/importance_sampling_ratio/mean": 0.3757447302341461, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.456852912902832, "sampling/sampling_logp_difference/mean": 0.012592746876180172, "step": 852, "step_time": 73.35480826650746 }, { "clip_ratio/high_max": 0.014387009316124022, "clip_ratio/high_mean": 0.004422408703248948, "clip_ratio/low_mean": 0.0030273437878349796, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007449752476532012, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 907.125, "completions/mean_terminated_length": 690.0714721679688, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "entropy": 0.14023137465119362, "epoch": 0.4852104664391354, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030122049152851105, "kl": 0.012433627620339394, "learning_rate": 4.638225255972697e-06, "loss": -0.005, "num_tokens": 90402368.0, "reward": 0.625, "reward_std": 0.896932065486908, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.1260628700256348, "sampling/importance_sampling_ratio/mean": 0.3464932441711426, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7850797176361084, "sampling/sampling_logp_difference/mean": 0.014382928609848022, "step": 853, "step_time": 74.6426230177749 }, { "clip_ratio/high_max": 0.012848522397689521, "clip_ratio/high_mean": 0.003950883721699938, "clip_ratio/low_mean": 0.0016113281380967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005562211794313043, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 966.0, "completions/mean_length": 833.9249877929688, "completions/mean_terminated_length": 661.952392578125, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "entropy": 0.11545874830335379, "epoch": 0.4857792946530148, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02179209515452385, "kl": 0.016324221156537533, "learning_rate": 4.633105802047782e-06, "loss": -0.0066, "num_tokens": 90502573.0, "reward": 1.024999976158142, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.4798951148986816, "sampling/importance_sampling_ratio/mean": 0.26765307784080505, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4552409648895264, "sampling/sampling_logp_difference/mean": 0.014605452306568623, "step": 854, "step_time": 72.84649598109536 }, { "clip_ratio/high_max": 0.009561704064253718, "clip_ratio/high_mean": 0.0032455738983117044, "clip_ratio/low_mean": 0.0016601562674622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004905730107566342, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 945.0, "completions/mean_length": 889.5250244140625, "completions/mean_terminated_length": 575.75, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.11078024096786976, "epoch": 0.4863481228668942, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030673876404762268, "kl": 0.009315552015323192, "learning_rate": 4.627986348122867e-06, "loss": 0.0155, "num_tokens": 90608162.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.6868865489959717, "sampling/importance_sampling_ratio/mean": 0.46269136667251587, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5493924617767334, "sampling/sampling_logp_difference/mean": 0.01248024869710207, "step": 855, "step_time": 72.23243645275943 }, { "clip_ratio/high_max": 0.0037198266945779324, "clip_ratio/high_mean": 0.0007439653563778847, "clip_ratio/low_mean": 0.00043945313518634066, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011834184915642254, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 892.4000244140625, "completions/mean_terminated_length": 497.6000061035156, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.13965409994125366, "epoch": 0.4869169510807736, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.008762877434492111, "kl": 0.011599867604672909, "learning_rate": 4.622866894197952e-06, "loss": -0.0072, "num_tokens": 90716074.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.4313547611236572, "sampling/importance_sampling_ratio/mean": 0.41604700684547424, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.46927547454834, "sampling/sampling_logp_difference/mean": 0.014621985144913197, "step": 856, "step_time": 73.1314155056607 }, { "clip_ratio/high_max": 0.010809863451868296, "clip_ratio/high_mean": 0.0030251311836764216, "clip_ratio/low_mean": 0.0018554687703726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004880599968601018, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 822.0, "completions/mean_length": 907.4000244140625, "completions/mean_terminated_length": 600.0, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "entropy": 0.09901680331677198, "epoch": 0.48748577929465303, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019261149689555168, "kl": 0.011053062858991325, "learning_rate": 4.617747440273038e-06, "loss": 0.0156, "num_tokens": 90824730.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 0.8743714094161987, "sampling/importance_sampling_ratio/mean": 0.2192600518465042, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8691184520721436, "sampling/sampling_logp_difference/mean": 0.011758306995034218, "step": 857, "step_time": 72.6407635321375 }, { "clip_ratio/high_max": 0.014687724993564188, "clip_ratio/high_mean": 0.00535246409708634, "clip_ratio/low_mean": 0.001342290677712299, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006694754702039063, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 798.0, "completions/mean_length": 726.8250122070312, "completions/mean_terminated_length": 566.8077392578125, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "entropy": 0.09727793186903, "epoch": 0.4880546075085324, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014149191789329052, "kl": 0.015628319117240608, "learning_rate": 4.612627986348123e-06, "loss": 0.0307, "num_tokens": 90928363.0, "reward": 1.125, "reward_std": 0.9111137390136719, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.5418200492858887, "sampling/importance_sampling_ratio/mean": 0.33946657180786133, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.33441162109375, "sampling/sampling_logp_difference/mean": 0.013868910260498524, "step": 858, "step_time": 72.13708640495315 }, { "clip_ratio/high_max": 0.01757132261991501, "clip_ratio/high_mean": 0.007151309488108382, "clip_ratio/low_mean": 0.0016426451729785185, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008793954679276794, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 586.7999877929688, "completions/mean_terminated_length": 459.8709411621094, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.11873070988804102, "epoch": 0.48862343572241185, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.035126037895679474, "kl": 0.020701143657788634, "learning_rate": 4.607508532423208e-06, "loss": -0.0185, "num_tokens": 91026227.0, "reward": 1.274999976158142, "reward_std": 0.8161038756370544, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.6425890922546387, "sampling/importance_sampling_ratio/mean": 0.4178217947483063, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.093118906021118, "sampling/sampling_logp_difference/mean": 0.01651277206838131, "step": 859, "step_time": 69.01568470639177 }, { "clip_ratio/high_max": 0.00943735201144591, "clip_ratio/high_mean": 0.0018874704546760768, "clip_ratio/low_mean": 0.0013916015523136593, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032790718832984567, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 784.0, "completions/mean_length": 957.625, "completions/mean_terminated_length": 644.7142944335938, "completions/min_length": 404.0, "completions/min_terminated_length": 404.0, "entropy": 0.08092163503170013, "epoch": 0.4891922639362912, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.031589217483997345, "kl": 0.009662732074502856, "learning_rate": 4.602389078498293e-06, "loss": -0.0705, "num_tokens": 91137020.0, "reward": 0.30000001192092896, "reward_std": 0.6868732571601868, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.6818244457244873, "sampling/importance_sampling_ratio/mean": 0.5555330514907837, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.884213924407959, "sampling/sampling_logp_difference/mean": 0.010689823888242245, "step": 860, "step_time": 75.70150256413035 }, { "clip_ratio/high_max": 0.010654563317075372, "clip_ratio/high_mean": 0.0032578071986790746, "clip_ratio/low_mean": 0.0014160156151774572, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004673822782933712, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 719.7750244140625, "completions/mean_terminated_length": 555.9615478515625, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.11148382443934679, "epoch": 0.48976109215017066, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022251972928643227, "kl": 0.014541179640218616, "learning_rate": 4.597269624573379e-06, "loss": 0.0105, "num_tokens": 91243643.0, "reward": 1.3000000715255737, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.9170432090759277, "sampling/importance_sampling_ratio/mean": 0.47915345430374146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1272926330566406, "sampling/sampling_logp_difference/mean": 0.015128130093216896, "step": 861, "step_time": 74.77679139468819 }, { "clip_ratio/high_max": 0.011818302271421999, "clip_ratio/high_mean": 0.0027502240845933557, "clip_ratio/low_mean": 0.0020019531075377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00475217736675404, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 961.0250244140625, "completions/mean_terminated_length": 664.1428833007812, "completions/min_length": 416.0, "completions/min_terminated_length": 416.0, "entropy": 0.1293050842359662, "epoch": 0.49032992036405004, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011470590718090534, "kl": 0.010708243353292346, "learning_rate": 4.592150170648465e-06, "loss": 0.0955, "num_tokens": 91353940.0, "reward": 0.32500001788139343, "reward_std": 0.7298577427864075, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.9836575984954834, "sampling/importance_sampling_ratio/mean": 0.41325196623802185, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.82531476020813, "sampling/sampling_logp_difference/mean": 0.014135653153061867, "step": 862, "step_time": 76.72512025409378 }, { "clip_ratio/high_max": 0.007765076821669936, "clip_ratio/high_mean": 0.0015530153759755194, "clip_ratio/low_mean": 0.0006103515552240424, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002163366931199562, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 895.0, "completions/mean_length": 913.5499877929688, "completions/mean_terminated_length": 655.8333740234375, "completions/min_length": 462.0, "completions/min_terminated_length": 462.0, "entropy": 0.162389500066638, "epoch": 0.4908987485779295, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.01220366545021534, "kl": 0.010373871307820082, "learning_rate": 4.5870307167235494e-06, "loss": -0.0143, "num_tokens": 91466642.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.5939948558807373, "sampling/importance_sampling_ratio/mean": 0.5270706415176392, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.377816081047058, "sampling/sampling_logp_difference/mean": 0.016058018431067467, "step": 863, "step_time": 75.95495951548219 }, { "clip_ratio/high_max": 0.013185939809773117, "clip_ratio/high_mean": 0.0038042779997340403, "clip_ratio/low_mean": 0.0033408233866794035, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007145101262722164, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 878.2000122070312, "completions/mean_terminated_length": 607.4285888671875, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.0972029035910964, "epoch": 0.49146757679180886, "frac_reward_zero_std": 0.0, "grad_norm": 0.03058447875082493, "kl": 0.010599554749205709, "learning_rate": 4.581911262798635e-06, "loss": 0.0518, "num_tokens": 91564466.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.7597386837005615, "sampling/importance_sampling_ratio/mean": 0.5707588195800781, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4554905891418457, "sampling/sampling_logp_difference/mean": 0.012351957149803638, "step": 864, "step_time": 72.80691304407082 }, { "clip_ratio/high_max": 0.016551434295251966, "clip_ratio/high_mean": 0.00586454471340403, "clip_ratio/low_mean": 0.0012422975414665416, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007106842298526317, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 825.0, "completions/mean_length": 755.75, "completions/mean_terminated_length": 576.9166870117188, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.07596648531034589, "epoch": 0.4920364050056883, "frac_reward_zero_std": 0.0, "grad_norm": 0.02389347366988659, "kl": 0.012223429628647864, "learning_rate": 4.57679180887372e-06, "loss": 0.1651, "num_tokens": 91670768.0, "reward": 0.875, "reward_std": 0.8223637342453003, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.9831868410110474, "sampling/importance_sampling_ratio/mean": 0.40730977058410645, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7130467891693115, "sampling/sampling_logp_difference/mean": 0.011697481386363506, "step": 865, "step_time": 74.09494425239973 }, { "clip_ratio/high_max": 0.007139909081161022, "clip_ratio/high_mean": 0.0016673435311531648, "clip_ratio/low_mean": 0.0028320312703726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004499374743318185, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 975.0, "completions/mean_length": 995.3250122070312, "completions/mean_terminated_length": 832.8333740234375, "completions/min_length": 595.0, "completions/min_terminated_length": 595.0, "entropy": 0.16279460489749908, "epoch": 0.4926052332195677, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.010817532427608967, "kl": 0.011783099733293056, "learning_rate": 4.571672354948806e-06, "loss": 0.0323, "num_tokens": 91783989.0, "reward": 0.2750000059604645, "reward_std": 0.6788942813873291, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.216339349746704, "sampling/importance_sampling_ratio/mean": 0.3949761688709259, "sampling/importance_sampling_ratio/min": 0.0025265831500291824, "sampling/sampling_logp_difference/max": 1.9859035015106201, "sampling/sampling_logp_difference/mean": 0.016102682799100876, "step": 866, "step_time": 74.86852229316719 }, { "clip_ratio/high_max": 0.01713041018228978, "clip_ratio/high_mean": 0.007005163264693692, "clip_ratio/low_mean": 0.0009961141950043384, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008001277397852391, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 777.6000366210938, "completions/mean_terminated_length": 613.3333740234375, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.1306443903595209, "epoch": 0.4931740614334471, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03678734228014946, "kl": 0.015629549394361675, "learning_rate": 4.56655290102389e-06, "loss": -0.0643, "num_tokens": 91890517.0, "reward": 1.149999976158142, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.4677268266677856, "sampling/importance_sampling_ratio/mean": 0.25543519854545593, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9938011169433594, "sampling/sampling_logp_difference/mean": 0.015602955594658852, "step": 867, "step_time": 75.46114114718512 }, { "clip_ratio/high_max": 0.01850139012094587, "clip_ratio/high_mean": 0.005974642757792026, "clip_ratio/low_mean": 0.0022216797224245965, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0081963223638013, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 782.6500244140625, "completions/mean_terminated_length": 652.6923217773438, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.1658881362527609, "epoch": 0.4937428896473265, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.038887567818164825, "kl": 0.019881399464793503, "learning_rate": 4.561433447098976e-06, "loss": 0.0371, "num_tokens": 91990631.0, "reward": 1.1750000715255737, "reward_std": 0.9306049346923828, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.949822187423706, "sampling/importance_sampling_ratio/mean": 0.5286506414413452, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.328347682952881, "sampling/sampling_logp_difference/mean": 0.017868608236312866, "step": 868, "step_time": 73.09953128872439 }, { "clip_ratio/high_max": 0.018677592859603465, "clip_ratio/high_mean": 0.007443614478688687, "clip_ratio/low_mean": 0.0014453979820245877, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008889012271538377, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 702.6000366210938, "completions/mean_terminated_length": 547.8518676757812, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.09132063016295433, "epoch": 0.4943117178612059, "frac_reward_zero_std": 0.0, "grad_norm": 0.03438054397702217, "kl": 0.0168052283115685, "learning_rate": 4.556313993174062e-06, "loss": -0.0314, "num_tokens": 92091023.0, "reward": 1.3000000715255737, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.9604222774505615, "sampling/importance_sampling_ratio/mean": 0.4552640914916992, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1428797245025635, "sampling/sampling_logp_difference/mean": 0.012980607338249683, "step": 869, "step_time": 70.58301739115268 }, { "clip_ratio/high_max": 0.012056544539518654, "clip_ratio/high_mean": 0.0027634215366560966, "clip_ratio/low_mean": 0.001746994774293853, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0045104163873475045, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 878.0, "completions/mean_length": 899.6749877929688, "completions/mean_terminated_length": 526.7000122070312, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.09009533934295177, "epoch": 0.4948805460750853, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014373619109392166, "kl": 0.012130723684094846, "learning_rate": 4.5511945392491475e-06, "loss": 0.074, "num_tokens": 92203170.0, "reward": 0.45000001788139343, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.3982744216918945, "sampling/importance_sampling_ratio/mean": 0.5177224278450012, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7961792945861816, "sampling/sampling_logp_difference/mean": 0.0120377317070961, "step": 870, "step_time": 75.9406674054917 }, { "clip_ratio/high_max": 0.015347287262557074, "clip_ratio/high_mean": 0.0033361240857630037, "clip_ratio/low_mean": 0.0009986364930227865, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004334760582423769, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 788.0, "completions/mean_length": 851.9249877929688, "completions/mean_terminated_length": 532.357177734375, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.11825136095285416, "epoch": 0.49544937428896474, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.013983719050884247, "kl": 0.013496680883690715, "learning_rate": 4.546075085324232e-06, "loss": -0.0133, "num_tokens": 92301231.0, "reward": 0.550000011920929, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.096719264984131, "sampling/importance_sampling_ratio/mean": 0.3726804554462433, "sampling/importance_sampling_ratio/min": 0.0008428114815615118, "sampling/sampling_logp_difference/max": 1.9726314544677734, "sampling/sampling_logp_difference/mean": 0.014264424331486225, "step": 871, "step_time": 74.2352566209156 }, { "clip_ratio/high_max": 0.007801542873494327, "clip_ratio/high_mean": 0.001560308548505418, "clip_ratio/low_mean": 0.0009277343961002771, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024880429555196315, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 848.125, "completions/mean_terminated_length": 521.5, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.09082378912717104, "epoch": 0.4960182025028441, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011237072758376598, "kl": 0.012111520860344172, "learning_rate": 4.5409556313993175e-06, "loss": 0.0544, "num_tokens": 92408324.0, "reward": 0.699999988079071, "reward_std": 0.9660917520523071, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.7358653545379639, "sampling/importance_sampling_ratio/mean": 0.3671260476112366, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.464071273803711, "sampling/sampling_logp_difference/mean": 0.01241134013980627, "step": 872, "step_time": 72.87504553608596 }, { "clip_ratio/high_max": 0.012521820375695825, "clip_ratio/high_mean": 0.003581553144613281, "clip_ratio/low_mean": 0.0008300781264551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004411631234688684, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 726.2999877929688, "completions/mean_terminated_length": 482.727294921875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.08343069721013308, "epoch": 0.49658703071672355, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.05111292377114296, "kl": 0.014187791151925921, "learning_rate": 4.535836177474403e-06, "loss": 0.005, "num_tokens": 92505472.0, "reward": 1.024999976158142, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.901885747909546, "sampling/importance_sampling_ratio/mean": 0.4456082880496979, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3841192722320557, "sampling/sampling_logp_difference/mean": 0.011763359420001507, "step": 873, "step_time": 71.70608807122335 }, { "clip_ratio/high_max": 0.014134085271507502, "clip_ratio/high_mean": 0.004420838668011129, "clip_ratio/low_mean": 0.0032470702863065526, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007667908910661936, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 865.4750366210938, "completions/mean_terminated_length": 651.0, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.1403285162523389, "epoch": 0.49715585893060293, "frac_reward_zero_std": 0.0, "grad_norm": 0.02838805876672268, "kl": 0.01456271018832922, "learning_rate": 4.530716723549488e-06, "loss": -0.0943, "num_tokens": 92618107.0, "reward": 0.699999988079071, "reward_std": 0.8828856945037842, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 1.8262226581573486, "sampling/importance_sampling_ratio/mean": 0.39444583654403687, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.163494110107422, "sampling/sampling_logp_difference/mean": 0.015681559219956398, "step": 874, "step_time": 76.41531424876302 }, { "clip_ratio/high_max": 0.012457919481676072, "clip_ratio/high_mean": 0.004113911083550192, "clip_ratio/low_mean": 0.0019042968488065526, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006018207874149084, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 839.5750122070312, "completions/mean_terminated_length": 614.1666870117188, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.15212634578347206, "epoch": 0.49772468714448237, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03683093562722206, "kl": 0.014203882194124162, "learning_rate": 4.525597269624573e-06, "loss": -0.0768, "num_tokens": 92731818.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.3609681129455566, "sampling/importance_sampling_ratio/mean": 0.3689747452735901, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1508655548095703, "sampling/sampling_logp_difference/mean": 0.0164633821696043, "step": 875, "step_time": 76.70392780820839 }, { "clip_ratio/high_max": 0.012897518347017467, "clip_ratio/high_mean": 0.00495713712007273, "clip_ratio/low_mean": 0.0016357421518478077, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006592879362870008, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 881.0, "completions/mean_length": 853.4000244140625, "completions/mean_terminated_length": 644.888916015625, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "entropy": 0.12253001239150763, "epoch": 0.49829351535836175, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027610450983047485, "kl": 0.01180734863737598, "learning_rate": 4.520477815699658e-06, "loss": 0.0733, "num_tokens": 92836762.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.9740688800811768, "sampling/importance_sampling_ratio/mean": 0.40655118227005005, "sampling/importance_sampling_ratio/min": 0.004530089441686869, "sampling/sampling_logp_difference/max": 2.3555281162261963, "sampling/sampling_logp_difference/mean": 0.014752394519746304, "step": 876, "step_time": 72.78662338666618 }, { "clip_ratio/high_max": 0.0055507568176835775, "clip_ratio/high_mean": 0.001228916211402975, "clip_ratio/low_mean": 0.002490234372089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003719150583492592, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 897.0, "completions/mean_length": 880.1500244140625, "completions/mean_terminated_length": 581.3846435546875, "completions/min_length": 421.0, "completions/min_terminated_length": 421.0, "entropy": 0.10419724881649017, "epoch": 0.4988623435722412, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022424090653657913, "kl": 0.009663004544563591, "learning_rate": 4.515358361774745e-06, "loss": -0.0072, "num_tokens": 92947448.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.6971303224563599, "sampling/importance_sampling_ratio/mean": 0.32194334268569946, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7735648155212402, "sampling/sampling_logp_difference/mean": 0.01228969357907772, "step": 877, "step_time": 74.9489105201792 }, { "clip_ratio/high_max": 0.008652489399537444, "clip_ratio/high_mean": 0.0023491100582759827, "clip_ratio/low_mean": 0.0020231904563843273, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004372300551040098, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 926.25, "completions/mean_terminated_length": 589.5555419921875, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "entropy": 0.10540108941495419, "epoch": 0.49943117178612056, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0072321537882089615, "kl": 0.010699208709411323, "learning_rate": 4.51023890784983e-06, "loss": 0.0277, "num_tokens": 93063962.0, "reward": 0.42500001192092896, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 0.9119437336921692, "sampling/importance_sampling_ratio/mean": 0.22519426047801971, "sampling/importance_sampling_ratio/min": 0.0013512325240299106, "sampling/sampling_logp_difference/max": 1.9808056354522705, "sampling/sampling_logp_difference/mean": 0.012511523440480232, "step": 878, "step_time": 77.85573266586289 }, { "clip_ratio/high_max": 0.012738800724036992, "clip_ratio/high_mean": 0.003803194354986772, "clip_ratio/low_mean": 0.0008300781519210432, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004633272532373667, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 950.0, "completions/mean_length": 888.4249877929688, "completions/mean_terminated_length": 606.84619140625, "completions/min_length": 394.0, "completions/min_terminated_length": 394.0, "entropy": 0.08027018839493394, "epoch": 0.5, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015469763427972794, "kl": 0.009067111590411514, "learning_rate": 4.505119453924915e-06, "loss": 0.0478, "num_tokens": 93169995.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.6109434366226196, "sampling/importance_sampling_ratio/mean": 0.3053859770298004, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0925652980804443, "sampling/sampling_logp_difference/mean": 0.010641521774232388, "step": 879, "step_time": 73.9486141782254 }, { "clip_ratio/high_max": 0.01459181250538677, "clip_ratio/high_mean": 0.005576085124630481, "clip_ratio/low_mean": 0.0018310547111468622, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00740713975392282, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 805.5499877929688, "completions/mean_terminated_length": 607.90478515625, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.07328821765258908, "epoch": 0.5005688282138794, "frac_reward_zero_std": 0.0, "grad_norm": 0.06481877714395523, "kl": 0.01617299218196422, "learning_rate": 4.5e-06, "loss": 0.0074, "num_tokens": 93264225.0, "reward": 0.9000000357627869, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.555809259414673, "sampling/importance_sampling_ratio/mean": 0.5214422941207886, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.7423510551452637, "sampling/sampling_logp_difference/mean": 0.011087299324572086, "step": 880, "step_time": 69.78914484987035 }, { "clip_ratio/high_max": 0.012051890138536692, "clip_ratio/high_mean": 0.0031360815337393433, "clip_ratio/low_mean": 0.0009138218265434261, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004049903363920748, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 713.1000366210938, "completions/mean_terminated_length": 545.6923217773438, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.10427819844335318, "epoch": 0.5011376564277589, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.038184333592653275, "kl": 0.029416995705105364, "learning_rate": 4.4948805460750855e-06, "loss": 0.0036, "num_tokens": 93371021.0, "reward": 1.274999976158142, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.344731569290161, "sampling/importance_sampling_ratio/mean": 0.4518265724182129, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0983786582946777, "sampling/sampling_logp_difference/mean": 0.014921069145202637, "step": 881, "step_time": 69.80022411234677 }, { "clip_ratio/high_max": 0.007090004626661539, "clip_ratio/high_mean": 0.0014180009457049891, "clip_ratio/low_mean": 0.0019177309586666524, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033357318461639807, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 930.4750366210938, "completions/mean_terminated_length": 556.375, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.11566482670605183, "epoch": 0.5017064846416383, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016769973561167717, "kl": 0.025223770295269787, "learning_rate": 4.489761092150171e-06, "loss": -0.0217, "num_tokens": 93480088.0, "reward": 0.30000001192092896, "reward_std": 0.6868732571601868, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 1.7791374921798706, "sampling/importance_sampling_ratio/mean": 0.2824110686779022, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4543709754943848, "sampling/sampling_logp_difference/mean": 0.014980750158429146, "step": 882, "step_time": 73.85283868806437 }, { "clip_ratio/high_max": 0.011205184855498374, "clip_ratio/high_mean": 0.0030001805862411857, "clip_ratio/low_mean": 0.002246093761641532, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005246274289675057, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 780.0, "completions/mean_length": 939.7250366210938, "completions/mean_terminated_length": 602.625, "completions/min_length": 407.0, "completions/min_terminated_length": 407.0, "entropy": 0.13296837359666824, "epoch": 0.5022753128555176, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014364011585712433, "kl": 0.013731099548749626, "learning_rate": 4.4846416382252556e-06, "loss": 0.0302, "num_tokens": 93591957.0, "reward": 0.4000000059604645, "reward_std": 0.8101914525032043, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.98162841796875, "sampling/importance_sampling_ratio/mean": 0.25453391671180725, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9752200841903687, "sampling/sampling_logp_difference/mean": 0.014769804663956165, "step": 883, "step_time": 76.2377800939139 }, { "clip_ratio/high_max": 0.01393041736446321, "clip_ratio/high_mean": 0.00541632941167336, "clip_ratio/low_mean": 0.003134815691737458, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008551145088858902, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 872.8500366210938, "completions/mean_terminated_length": 721.7000122070312, "completions/min_length": 457.0, "completions/min_terminated_length": 457.0, "entropy": 0.1130337743088603, "epoch": 0.502844141069397, "frac_reward_zero_std": 0.0, "grad_norm": 0.0185768473893404, "kl": 0.0139590137405321, "learning_rate": 4.479522184300342e-06, "loss": -0.0058, "num_tokens": 93697399.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.7422651052474976, "sampling/importance_sampling_ratio/mean": 0.3159481883049011, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.516254425048828, "sampling/sampling_logp_difference/mean": 0.014761045575141907, "step": 884, "step_time": 75.73353804415092 }, { "clip_ratio/high_max": 0.008630802563857287, "clip_ratio/high_mean": 0.002912001851655077, "clip_ratio/low_mean": 0.0031380355067085475, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.006050037278328091, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 892.4500122070312, "completions/mean_terminated_length": 673.2000122070312, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "entropy": 0.09810192510485649, "epoch": 0.5034129692832765, "frac_reward_zero_std": 0.0, "grad_norm": 0.025565236806869507, "kl": 0.008563763345591724, "learning_rate": 4.4744027303754264e-06, "loss": -0.0386, "num_tokens": 93816881.0, "reward": 0.699999988079071, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.084900140762329, "sampling/importance_sampling_ratio/mean": 0.4578956663608551, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.498703956604004, "sampling/sampling_logp_difference/mean": 0.012361413799226284, "step": 885, "step_time": 77.75797089678235 }, { "clip_ratio/high_max": 0.013565446133725345, "clip_ratio/high_mean": 0.003880901320371777, "clip_ratio/low_mean": 0.00029296874708961695, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004173870023805648, "completions/clipped_ratio": 0.05000000074505806, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 602.4000244140625, "completions/mean_terminated_length": 580.2105102539062, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.11373486462980509, "epoch": 0.5039817974971559, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011907813139259815, "kl": 0.016432040894869715, "learning_rate": 4.469283276450512e-06, "loss": -0.0377, "num_tokens": 93914681.0, "reward": 1.899999976158142, "reward_std": 0.4414428472518921, "rewards/format_reward/mean": 0.949999988079071, "rewards/format_reward/std": 0.22072143852710724, "rewards/gpt_judge_reward/mean": 0.949999988079071, "rewards/gpt_judge_reward/std": 0.22072143852710724, "sampling/importance_sampling_ratio/max": 2.4243087768554688, "sampling/importance_sampling_ratio/mean": 0.4584026336669922, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1188175678253174, "sampling/sampling_logp_difference/mean": 0.015318537130951881, "step": 886, "step_time": 71.9034894707147 }, { "clip_ratio/high_max": 0.018772140028886497, "clip_ratio/high_mean": 0.007364379125647247, "clip_ratio/low_mean": 0.0008789062449068297, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00824328523594886, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 786.9750366210938, "completions/mean_terminated_length": 628.9583740234375, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "entropy": 0.11457353364676237, "epoch": 0.5045506257110353, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030358437448740005, "kl": 0.013026004424318671, "learning_rate": 4.464163822525597e-06, "loss": 0.005, "num_tokens": 94025144.0, "reward": 0.925000011920929, "reward_std": 0.8589648008346558, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.4239022731781006, "sampling/importance_sampling_ratio/mean": 0.33232346177101135, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.464240312576294, "sampling/sampling_logp_difference/mean": 0.014706087298691273, "step": 887, "step_time": 74.68042043526657 }, { "clip_ratio/high_max": 0.0068342497397679836, "clip_ratio/high_mean": 0.002231885118817445, "clip_ratio/low_mean": 0.0022460937325377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004477978887734935, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 952.0, "completions/mean_length": 952.5499877929688, "completions/mean_terminated_length": 738.2000122070312, "completions/min_length": 549.0, "completions/min_terminated_length": 549.0, "entropy": 0.11662244331091642, "epoch": 0.5051194539249146, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03207892179489136, "kl": 0.010634314792696387, "learning_rate": 4.459044368600683e-06, "loss": -0.113, "num_tokens": 94131822.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.4796475172042847, "sampling/importance_sampling_ratio/mean": 0.2662689685821533, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.886441946029663, "sampling/sampling_logp_difference/mean": 0.013766154646873474, "step": 888, "step_time": 74.8458319969941 }, { "clip_ratio/high_max": 0.008997673983685672, "clip_ratio/high_mean": 0.0019285338057670742, "clip_ratio/low_mean": 0.0010253906184516381, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002953924413304776, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 969.0, "completions/mean_length": 972.2250366210938, "completions/mean_terminated_length": 678.8333740234375, "completions/min_length": 504.0, "completions/min_terminated_length": 504.0, "entropy": 0.10171247739344835, "epoch": 0.5056882821387941, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.0075711882673203945, "kl": 0.009861031780019403, "learning_rate": 4.453924914675768e-06, "loss": 0.0341, "num_tokens": 94244975.0, "reward": 0.30000001192092896, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 1.7167390584945679, "sampling/importance_sampling_ratio/mean": 0.24546575546264648, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.699787139892578, "sampling/sampling_logp_difference/mean": 0.012433554045855999, "step": 889, "step_time": 76.55343447020277 }, { "clip_ratio/high_max": 0.015386981365736574, "clip_ratio/high_mean": 0.006677322220639326, "clip_ratio/low_mean": 0.003596205817302689, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010273528168909252, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 789.8250122070312, "completions/mean_terminated_length": 616.7391357421875, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.16091403178870678, "epoch": 0.5062571103526735, "frac_reward_zero_std": 0.0, "grad_norm": 0.029098309576511383, "kl": 0.017319972161203623, "learning_rate": 4.448805460750854e-06, "loss": 0.0419, "num_tokens": 94346760.0, "reward": 1.024999976158142, "reward_std": 0.9996795058250427, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.679513454437256, "sampling/importance_sampling_ratio/mean": 0.4251921772956848, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7713112831115723, "sampling/sampling_logp_difference/mean": 0.017934294417500496, "step": 890, "step_time": 76.09767940384336 }, { "clip_ratio/high_max": 0.01563997141784057, "clip_ratio/high_mean": 0.007261557795573026, "clip_ratio/low_mean": 0.0012207031322759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008482260920573026, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 776.7000122070312, "completions/mean_terminated_length": 682.8965454101562, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.06661028368398547, "epoch": 0.5068259385665529, "frac_reward_zero_std": 0.0, "grad_norm": 0.0349610336124897, "kl": 0.011072527267970145, "learning_rate": 4.443686006825938e-06, "loss": 0.12, "num_tokens": 94460596.0, "reward": 1.4500000476837158, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.7250000238418579, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.734745502471924, "sampling/importance_sampling_ratio/mean": 0.45020121335983276, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7038815021514893, "sampling/sampling_logp_difference/mean": 0.010556042194366455, "step": 891, "step_time": 78.52254387899302 }, { "clip_ratio/high_max": 0.013955381233245134, "clip_ratio/high_mean": 0.004548747034277767, "clip_ratio/low_mean": 0.0008789062503637979, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005427653348306194, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 908.0, "completions/mean_length": 749.1000366210938, "completions/mean_terminated_length": 565.8333740234375, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.12907140981405973, "epoch": 0.5073947667804323, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02419198304414749, "kl": 0.014648021664470434, "learning_rate": 4.4385665529010245e-06, "loss": 0.0033, "num_tokens": 94557360.0, "reward": 1.100000023841858, "reward_std": 0.9554164409637451, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.140036106109619, "sampling/importance_sampling_ratio/mean": 0.35005879402160645, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.024704933166504, "sampling/sampling_logp_difference/mean": 0.015773847699165344, "step": 892, "step_time": 70.1227248844225 }, { "clip_ratio/high_max": 0.015366607694886625, "clip_ratio/high_mean": 0.005209250375628471, "clip_ratio/low_mean": 0.0018492368544684723, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00705848733196035, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 868.5, "completions/mean_terminated_length": 678.4444580078125, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.1210529962554574, "epoch": 0.5079635949943118, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02253355085849762, "kl": 0.011794605874456465, "learning_rate": 4.433447098976109e-06, "loss": 0.0269, "num_tokens": 94666652.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.8375604152679443, "sampling/importance_sampling_ratio/mean": 0.4861184060573578, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.35775089263916, "sampling/sampling_logp_difference/mean": 0.013825413770973682, "step": 893, "step_time": 75.11374809592962 }, { "clip_ratio/high_max": 0.014463573694229126, "clip_ratio/high_mean": 0.0030978952418081462, "clip_ratio/low_mean": 0.0006103515734139364, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037082467752043158, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 827.4000244140625, "completions/mean_terminated_length": 561.4117431640625, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.115269361063838, "epoch": 0.5085324232081911, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.007199384272098541, "kl": 0.014659061445854604, "learning_rate": 4.4283276450511945e-06, "loss": 0.0035, "num_tokens": 94768996.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.098224401473999, "sampling/importance_sampling_ratio/mean": 0.4332287907600403, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9473061561584473, "sampling/sampling_logp_difference/mean": 0.014210629276931286, "step": 894, "step_time": 71.22053920989856 }, { "clip_ratio/high_max": 0.020498899277299643, "clip_ratio/high_mean": 0.009647306986153126, "clip_ratio/low_mean": 0.0011732987222785596, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010820605792105198, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.0, "completions/mean_length": 723.1000366210938, "completions/mean_terminated_length": 522.5, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.13382465858012438, "epoch": 0.5091012514220705, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02262934111058712, "kl": 0.015709774801507592, "learning_rate": 4.42320819112628e-06, "loss": 0.0316, "num_tokens": 94874232.0, "reward": 1.1750000715255737, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.9692914485931396, "sampling/importance_sampling_ratio/mean": 0.31808406114578247, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8759132623672485, "sampling/sampling_logp_difference/mean": 0.0167413130402565, "step": 895, "step_time": 72.36856770189479 }, { "clip_ratio/high_max": 0.013640314049553126, "clip_ratio/high_mean": 0.0038106114079710096, "clip_ratio/low_mean": 0.0007812500170985004, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004591861419612542, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 938.0, "completions/mean_length": 777.1000366210938, "completions/mean_terminated_length": 612.5, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.11077573522925377, "epoch": 0.5096700796359499, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014264076016843319, "kl": 0.01911084831226617, "learning_rate": 4.418088737201365e-06, "loss": -0.0037, "num_tokens": 94983988.0, "reward": 1.0, "reward_std": 0.9058216214179993, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.121375799179077, "sampling/importance_sampling_ratio/mean": 0.290479838848114, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3916637897491455, "sampling/sampling_logp_difference/mean": 0.014217889867722988, "step": 896, "step_time": 75.20602403418161 }, { "clip_ratio/high_max": 0.01378132589161396, "clip_ratio/high_mean": 0.005393099971115589, "clip_ratio/low_mean": 0.0016357422082364792, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00702884208294563, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 841.7000122070312, "completions/mean_terminated_length": 568.25, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.09786995872855186, "epoch": 0.5102389078498294, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03142070025205612, "kl": 0.016365260118618608, "learning_rate": 4.412969283276451e-06, "loss": 0.0141, "num_tokens": 95078520.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.674488067626953, "sampling/importance_sampling_ratio/mean": 0.4025156497955322, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.194746255874634, "sampling/sampling_logp_difference/mean": 0.01301133818924427, "step": 897, "step_time": 69.8883737407159 }, { "clip_ratio/high_max": 0.010762945108581334, "clip_ratio/high_mean": 0.0031551895226584747, "clip_ratio/low_mean": 0.0019042968924622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005059486458776519, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 939.0, "completions/mean_length": 883.375, "completions/mean_terminated_length": 649.0000610351562, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "entropy": 0.0915987091138959, "epoch": 0.5108077360637088, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03411555290222168, "kl": 0.012130052491556853, "learning_rate": 4.407849829351536e-06, "loss": -0.1067, "num_tokens": 95185183.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.6694495677948, "sampling/importance_sampling_ratio/mean": 0.5013224482536316, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7599866390228271, "sampling/sampling_logp_difference/mean": 0.011766709387302399, "step": 898, "step_time": 73.92194723291323 }, { "clip_ratio/high_max": 0.016495312680490315, "clip_ratio/high_mean": 0.004921684449072927, "clip_ratio/low_mean": 0.0029860138092772104, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007907698105555028, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 773.3500366210938, "completions/mean_terminated_length": 652.6666870117188, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.154269658960402, "epoch": 0.5113765642775882, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03826040029525757, "kl": 0.01334699906874448, "learning_rate": 4.402730375426621e-06, "loss": 0.1529, "num_tokens": 95293477.0, "reward": 1.25, "reward_std": 0.9268086552619934, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.9943084716796875, "sampling/importance_sampling_ratio/mean": 0.5711788535118103, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9112944602966309, "sampling/sampling_logp_difference/mean": 0.017195548862218857, "step": 899, "step_time": 74.14298478327692 }, { "clip_ratio/high_max": 0.014807250117883086, "clip_ratio/high_mean": 0.0067895318497903645, "clip_ratio/low_mean": 0.0014754780822840985, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00826500979019329, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 772.2000122070312, "completions/mean_terminated_length": 688.2667236328125, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.12344012223184109, "epoch": 0.5119453924914675, "frac_reward_zero_std": 0.0, "grad_norm": 0.03407662361860275, "kl": 0.01558527909219265, "learning_rate": 4.397610921501707e-06, "loss": 0.0406, "num_tokens": 95385557.0, "reward": 1.274999976158142, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.7855544090270996, "sampling/importance_sampling_ratio/mean": 0.44458526372909546, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8856797218322754, "sampling/sampling_logp_difference/mean": 0.015444395132362843, "step": 900, "step_time": 70.09900534688495 }, { "clip_ratio/high_max": 0.01895581732969731, "clip_ratio/high_mean": 0.007286011066753417, "clip_ratio/low_mean": 0.0013671875130967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008653198601678014, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 956.0, "completions/mean_length": 632.6500244140625, "completions/mean_terminated_length": 519.0322265625, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.1162380538880825, "epoch": 0.512514220705347, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018397321924567223, "kl": 0.019071071408689022, "learning_rate": 4.392491467576792e-06, "loss": -0.0079, "num_tokens": 95479103.0, "reward": 1.399999976158142, "reward_std": 0.8412445187568665, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.5111991167068481, "sampling/importance_sampling_ratio/mean": 0.2756044864654541, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5109195709228516, "sampling/sampling_logp_difference/mean": 0.015842607244849205, "step": 901, "step_time": 70.3607912780717 }, { "clip_ratio/high_max": 0.01687626022612676, "clip_ratio/high_mean": 0.0050582209369167686, "clip_ratio/low_mean": 0.0012939453372382559, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006352166295982897, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 889.0, "completions/mean_length": 826.0499877929688, "completions/mean_terminated_length": 607.26318359375, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.12026916816830635, "epoch": 0.5130830489192264, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03894411399960518, "kl": 0.01309986540582031, "learning_rate": 4.387372013651877e-06, "loss": 0.0347, "num_tokens": 95591817.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.321293592453003, "sampling/importance_sampling_ratio/mean": 0.44690603017807007, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3386306762695312, "sampling/sampling_logp_difference/mean": 0.014417359605431557, "step": 902, "step_time": 76.87968455487862 }, { "clip_ratio/high_max": 0.015968250227160752, "clip_ratio/high_mean": 0.005879057716811076, "clip_ratio/low_mean": 0.001694983227935154, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007574040908366442, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 749.0, "completions/mean_terminated_length": 584.0, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 0.1388279814273119, "epoch": 0.5136518771331058, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.032047517597675323, "kl": 0.02476907556410879, "learning_rate": 4.3822525597269626e-06, "loss": -0.1246, "num_tokens": 95692057.0, "reward": 1.125, "reward_std": 0.9388344883918762, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.9891815185546875, "sampling/importance_sampling_ratio/mean": 0.3422449231147766, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.905027151107788, "sampling/sampling_logp_difference/mean": 0.01708238199353218, "step": 903, "step_time": 70.6233563257847 }, { "clip_ratio/high_max": 0.012678617669735104, "clip_ratio/high_mean": 0.003296033450169489, "clip_ratio/low_mean": 0.002047776593826711, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00534381007310003, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 764.375, "completions/mean_terminated_length": 551.95458984375, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.15896603278815746, "epoch": 0.5142207053469852, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0342600978910923, "kl": 0.013782266178168356, "learning_rate": 4.377133105802048e-06, "loss": -0.027, "num_tokens": 95803016.0, "reward": 1.024999976158142, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.647491455078125, "sampling/importance_sampling_ratio/mean": 0.5191625356674194, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1370644569396973, "sampling/sampling_logp_difference/mean": 0.01783205196261406, "step": 904, "step_time": 73.08865762199275 }, { "clip_ratio/high_max": 0.010059314430691302, "clip_ratio/high_mean": 0.0027927025803364813, "clip_ratio/low_mean": 0.0021840284898644313, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004976731113856658, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 916.125, "completions/mean_terminated_length": 736.3333740234375, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "entropy": 0.08936378173530102, "epoch": 0.5147895335608647, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01654127798974514, "kl": 0.009699003072455525, "learning_rate": 4.372013651877133e-06, "loss": 0.0405, "num_tokens": 95896693.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 1.8766610622406006, "sampling/importance_sampling_ratio/mean": 0.3721945583820343, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8376590013504028, "sampling/sampling_logp_difference/mean": 0.011708910576999187, "step": 905, "step_time": 70.48161507025361 }, { "clip_ratio/high_max": 0.01728420879226178, "clip_ratio/high_mean": 0.005199823441216722, "clip_ratio/low_mean": 0.0025824270996963605, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007782250439049676, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 839.0, "completions/mean_length": 656.2000122070312, "completions/mean_terminated_length": 564.25, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "entropy": 0.11444061808288097, "epoch": 0.515358361774744, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02828279696404934, "kl": 0.015906542423181236, "learning_rate": 4.366894197952219e-06, "loss": -0.0591, "num_tokens": 95998997.0, "reward": 1.375, "reward_std": 0.8066232204437256, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.3058977127075195, "sampling/importance_sampling_ratio/mean": 0.4098592698574066, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6316709518432617, "sampling/sampling_logp_difference/mean": 0.015660647302865982, "step": 906, "step_time": 70.8009364171885 }, { "clip_ratio/high_max": 0.012809256208129227, "clip_ratio/high_mean": 0.002941424841992557, "clip_ratio/low_mean": 0.0020996093517169356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005041034135501832, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 967.0, "completions/mean_length": 853.5750122070312, "completions/mean_terminated_length": 597.9375, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.1075269216671586, "epoch": 0.5159271899886234, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01782122440636158, "kl": 0.014814662747085094, "learning_rate": 4.3617747440273034e-06, "loss": 0.0574, "num_tokens": 96107844.0, "reward": 0.675000011920929, "reward_std": 0.8883144855499268, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.9935609102249146, "sampling/importance_sampling_ratio/mean": 0.4066007137298584, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.105602264404297, "sampling/sampling_logp_difference/mean": 0.01337472815066576, "step": 907, "step_time": 74.18247899203561 }, { "clip_ratio/high_max": 0.008399553247727454, "clip_ratio/high_mean": 0.0016799106524558738, "clip_ratio/low_mean": 0.00155661807366414, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032365286897402257, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 923.5499877929688, "completions/mean_terminated_length": 737.0000610351562, "completions/min_length": 440.0, "completions/min_terminated_length": 440.0, "entropy": 0.15876373182982206, "epoch": 0.5164960182025028, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.022432371973991394, "kl": 0.012210040120407939, "learning_rate": 4.35665529010239e-06, "loss": -0.0087, "num_tokens": 96212394.0, "reward": 0.675000011920929, "reward_std": 0.9442809820175171, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.084406852722168, "sampling/importance_sampling_ratio/mean": 0.3982832729816437, "sampling/importance_sampling_ratio/min": 0.0006713745533488691, "sampling/sampling_logp_difference/max": 3.0800986289978027, "sampling/sampling_logp_difference/mean": 0.015557381324470043, "step": 908, "step_time": 72.42686698469333 }, { "clip_ratio/high_max": 0.013033546740189195, "clip_ratio/high_mean": 0.0037085536459926516, "clip_ratio/low_mean": 0.0011718750101863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004880428547039628, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 859.1000366210938, "completions/mean_terminated_length": 709.90478515625, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.10480250511318445, "epoch": 0.5170648464163823, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.012541309930384159, "kl": 0.013789624674245715, "learning_rate": 4.351535836177474e-06, "loss": 0.0153, "num_tokens": 96317894.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.766141414642334, "sampling/importance_sampling_ratio/mean": 0.41224589943885803, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.207024574279785, "sampling/sampling_logp_difference/mean": 0.013654911890625954, "step": 909, "step_time": 74.04103116411716 }, { "clip_ratio/high_max": 0.012619501329027116, "clip_ratio/high_mean": 0.0028001433674944565, "clip_ratio/low_mean": 0.0017333984305878403, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004533541839919053, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 867.0, "completions/mean_length": 918.4000244140625, "completions/mean_terminated_length": 554.6666870117188, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.075260269921273, "epoch": 0.5176336746302617, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011292478069663048, "kl": 0.01217836473369971, "learning_rate": 4.34641638225256e-06, "loss": 0.0309, "num_tokens": 96429886.0, "reward": 0.45000001788139343, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290207743644714, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 1.3876043558120728, "sampling/importance_sampling_ratio/mean": 0.22871167957782745, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.528956890106201, "sampling/sampling_logp_difference/mean": 0.01050725020468235, "step": 910, "step_time": 76.06365834898315 }, { "clip_ratio/high_max": 0.008444337319815531, "clip_ratio/high_mean": 0.00212247565650614, "clip_ratio/low_mean": 0.0012451171860448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003367592886206694, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 865.4249877929688, "completions/mean_terminated_length": 671.6111450195312, "completions/min_length": 424.0, "completions/min_terminated_length": 424.0, "entropy": 0.12677060719579458, "epoch": 0.518202502844141, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03411104902625084, "kl": 0.010925181151833385, "learning_rate": 4.341296928327645e-06, "loss": -0.1186, "num_tokens": 96529039.0, "reward": 0.675000011920929, "reward_std": 0.8285761475563049, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.9373955726623535, "sampling/importance_sampling_ratio/mean": 0.38524460792541504, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3100396394729614, "sampling/sampling_logp_difference/mean": 0.013755452819168568, "step": 911, "step_time": 70.43698744289577 }, { "clip_ratio/high_max": 0.010300436289981008, "clip_ratio/high_mean": 0.0028044199862051755, "clip_ratio/low_mean": 0.001904296877910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004708716849563643, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 923.0750122070312, "completions/mean_terminated_length": 657.0, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.07531606126576662, "epoch": 0.5187713310580204, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01019712071865797, "kl": 0.010619198554195464, "learning_rate": 4.336177474402731e-06, "loss": -0.0098, "num_tokens": 96637282.0, "reward": 0.45000001788139343, "reward_std": 0.7828285098075867, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.3906967639923096, "sampling/importance_sampling_ratio/mean": 0.5280145406723022, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9744741916656494, "sampling/sampling_logp_difference/mean": 0.010484687983989716, "step": 912, "step_time": 74.35750123881735 }, { "clip_ratio/high_max": 0.01334976073121652, "clip_ratio/high_mean": 0.003607379665481858, "clip_ratio/low_mean": 0.0014722295109095285, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005079609167296439, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 949.0, "completions/mean_length": 910.7999877929688, "completions/mean_terminated_length": 646.6666870117188, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.1292245825752616, "epoch": 0.5193401592718999, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016021305695176125, "kl": 0.013011840637773275, "learning_rate": 4.331058020477816e-06, "loss": 0.0545, "num_tokens": 96750362.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.718161106109619, "sampling/importance_sampling_ratio/mean": 0.3614121973514557, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.498704195022583, "sampling/sampling_logp_difference/mean": 0.014759850688278675, "step": 913, "step_time": 76.96683878987096 }, { "clip_ratio/high_max": 0.0019751694053411484, "clip_ratio/high_mean": 0.00039503388688899577, "clip_ratio/low_mean": 0.00041503906686557457, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008100729537545703, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 914.875, "completions/mean_terminated_length": 539.0, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.13351105339825153, "epoch": 0.5199089874857793, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.011406551115214825, "kl": 0.013526084716431797, "learning_rate": 4.3259385665529015e-06, "loss": 0.0377, "num_tokens": 96848677.0, "reward": 0.42500001192092896, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.7367215156555176, "sampling/importance_sampling_ratio/mean": 0.4648555815219879, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2031502723693848, "sampling/sampling_logp_difference/mean": 0.01491682231426239, "step": 914, "step_time": 72.05830743862316 }, { "clip_ratio/high_max": 0.016968953306786716, "clip_ratio/high_mean": 0.0058178475592285395, "clip_ratio/low_mean": 0.0027752184905693866, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008593066246248782, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 852.0, "completions/mean_length": 727.5750122070312, "completions/mean_terminated_length": 485.04547119140625, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.10882399510592222, "epoch": 0.5204778156996587, "frac_reward_zero_std": 0.0, "grad_norm": 0.029557742178440094, "kl": 0.013819208252243698, "learning_rate": 4.320819112627986e-06, "loss": 0.017, "num_tokens": 96952724.0, "reward": 0.9750000238418579, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.9354686737060547, "sampling/importance_sampling_ratio/mean": 0.3834843039512634, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9116969108581543, "sampling/sampling_logp_difference/mean": 0.01407652534544468, "step": 915, "step_time": 72.57593403337523 }, { "clip_ratio/high_max": 0.015331570291891694, "clip_ratio/high_mean": 0.006218321243068203, "clip_ratio/low_mean": 0.0016601562565483619, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007878477510530502, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 745.7750244140625, "completions/mean_terminated_length": 540.1304321289062, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.07245729304850101, "epoch": 0.5210466439135382, "frac_reward_zero_std": 0.0, "grad_norm": 0.04288385808467865, "kl": 0.013176617794670165, "learning_rate": 4.3156996587030715e-06, "loss": 0.3087, "num_tokens": 97044635.0, "reward": 1.100000023841858, "reward_std": 0.9818872213363647, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.9000966548919678, "sampling/importance_sampling_ratio/mean": 0.4965721666812897, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.809255599975586, "sampling/sampling_logp_difference/mean": 0.010464373044669628, "step": 916, "step_time": 75.92896780162118 }, { "clip_ratio/high_max": 0.009790311218239367, "clip_ratio/high_mean": 0.003269663022365421, "clip_ratio/low_mean": 0.0011474609418655746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004417124007886741, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 941.0, "completions/mean_length": 752.125, "completions/mean_terminated_length": 480.25, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.08494830317795277, "epoch": 0.5216154721274175, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02004634030163288, "kl": 0.015383603982627392, "learning_rate": 4.310580204778157e-06, "loss": 0.0172, "num_tokens": 97134104.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.5383925437927246, "sampling/importance_sampling_ratio/mean": 0.5291016101837158, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9704844951629639, "sampling/sampling_logp_difference/mean": 0.011760633438825607, "step": 917, "step_time": 71.01138524664566 }, { "clip_ratio/high_max": 0.0160918541951105, "clip_ratio/high_mean": 0.005533067072974518, "clip_ratio/low_mean": 0.0016357421591237653, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007168809126596898, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 839.0, "completions/mean_length": 799.5, "completions/mean_terminated_length": 596.3809814453125, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.10752841923385859, "epoch": 0.5221843003412969, "frac_reward_zero_std": 0.0, "grad_norm": 0.03250405564904213, "kl": 0.012803271180018783, "learning_rate": 4.305460750853242e-06, "loss": -0.0661, "num_tokens": 97239428.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.654026985168457, "sampling/importance_sampling_ratio/mean": 0.4719790518283844, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8856068849563599, "sampling/sampling_logp_difference/mean": 0.013387582264840603, "step": 918, "step_time": 73.99131972598843 }, { "clip_ratio/high_max": 0.015335196512751281, "clip_ratio/high_mean": 0.00602563310530968, "clip_ratio/low_mean": 0.0016790931877039839, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0077047262457199395, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 699.0750122070312, "completions/mean_terminated_length": 504.1199951171875, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.09587900061160326, "epoch": 0.5227531285551763, "frac_reward_zero_std": 0.0, "grad_norm": 0.034313976764678955, "kl": 0.01874101918656379, "learning_rate": 4.300341296928328e-06, "loss": -0.0953, "num_tokens": 97330631.0, "reward": 1.2000000476837158, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.5495662689208984, "sampling/importance_sampling_ratio/mean": 0.5797591209411621, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.221055030822754, "sampling/sampling_logp_difference/mean": 0.012429761700332165, "step": 919, "step_time": 68.32822599331848 }, { "clip_ratio/high_max": 0.014044072944670916, "clip_ratio/high_mean": 0.0037155795143917203, "clip_ratio/low_mean": 0.0013427734520519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005058352980995551, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 846.75, "completions/mean_terminated_length": 669.5, "completions/min_length": 393.0, "completions/min_terminated_length": 393.0, "entropy": 0.1219828650355339, "epoch": 0.5233219567690558, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01624452881515026, "kl": 0.012069852440617979, "learning_rate": 4.295221843003413e-06, "loss": -0.0083, "num_tokens": 97439053.0, "reward": 0.9000000357627869, "reward_std": 0.9554165005683899, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.0729877948760986, "sampling/importance_sampling_ratio/mean": 0.3869097828865051, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5747838020324707, "sampling/sampling_logp_difference/mean": 0.014116874895989895, "step": 920, "step_time": 74.79763909103349 }, { "clip_ratio/high_max": 0.012434350093826652, "clip_ratio/high_mean": 0.0038770315004512668, "clip_ratio/low_mean": 0.0018428171551931882, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005719848675653338, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 954.0, "completions/mean_length": 725.6000366210938, "completions/mean_terminated_length": 546.5599975585938, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.0911745335906744, "epoch": 0.5238907849829352, "frac_reward_zero_std": 0.0, "grad_norm": 0.03953370824456215, "kl": 0.011833888886030763, "learning_rate": 4.290102389078499e-06, "loss": 0.0434, "num_tokens": 97542069.0, "reward": 1.0, "reward_std": 0.8770580291748047, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.805511713027954, "sampling/importance_sampling_ratio/mean": 0.4205241799354553, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7861111164093018, "sampling/sampling_logp_difference/mean": 0.012177466414868832, "step": 921, "step_time": 72.98963907337748 }, { "clip_ratio/high_max": 0.010592721810098737, "clip_ratio/high_mean": 0.0023332682467298582, "clip_ratio/low_mean": 0.000982864637990133, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00331613291928079, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 960.0, "completions/mean_length": 793.875, "completions/mean_terminated_length": 512.6111450195312, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.10588854365050793, "epoch": 0.5244596131968146, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009896205738186836, "kl": 0.014213995076715946, "learning_rate": 4.284982935153583e-06, "loss": 0.0187, "num_tokens": 97647824.0, "reward": 0.824999988079071, "reward_std": 0.957761824131012, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.6677427291870117, "sampling/importance_sampling_ratio/mean": 0.2740621268749237, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.592953681945801, "sampling/sampling_logp_difference/mean": 0.014582893811166286, "step": 922, "step_time": 72.18920267629437 }, { "clip_ratio/high_max": 0.016849619569256902, "clip_ratio/high_mean": 0.007481138964067213, "clip_ratio/low_mean": 0.0021756079950137064, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00965674692997709, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 876.0, "completions/mean_length": 672.9750366210938, "completions/mean_terminated_length": 522.5357666015625, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.12456037010997534, "epoch": 0.5250284414106939, "frac_reward_zero_std": 0.0, "grad_norm": 0.03508097305893898, "kl": 0.015735827619209886, "learning_rate": 4.279863481228669e-06, "loss": -0.0546, "num_tokens": 97741647.0, "reward": 1.274999976158142, "reward_std": 0.905467689037323, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.435952663421631, "sampling/importance_sampling_ratio/mean": 0.5382552146911621, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2632787227630615, "sampling/sampling_logp_difference/mean": 0.015504169277846813, "step": 923, "step_time": 70.93959977896884 }, { "clip_ratio/high_max": 0.0045579299330711365, "clip_ratio/high_mean": 0.0011758565524360165, "clip_ratio/low_mean": 0.0028076171583961695, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003983473754487932, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 861.0, "completions/mean_length": 975.0250244140625, "completions/mean_terminated_length": 534.25, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.08163715526461601, "epoch": 0.5255972696245734, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01160310860723257, "kl": 0.009924498270265758, "learning_rate": 4.274744027303754e-06, "loss": 0.0592, "num_tokens": 97849280.0, "reward": 0.20000000298023224, "reward_std": 0.6076436042785645, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 1.801142692565918, "sampling/importance_sampling_ratio/mean": 0.33584851026535034, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3331878185272217, "sampling/sampling_logp_difference/mean": 0.010383948683738708, "step": 924, "step_time": 74.12937107798643 }, { "clip_ratio/high_max": 0.012790453154593706, "clip_ratio/high_mean": 0.004417411735630594, "clip_ratio/low_mean": 0.0014404296744032763, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005857841344550252, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 883.4000244140625, "completions/mean_terminated_length": 591.3846435546875, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "entropy": 0.08903631195425987, "epoch": 0.5261660978384528, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03884488344192505, "kl": 0.010408980946522206, "learning_rate": 4.2696245733788396e-06, "loss": -0.025, "num_tokens": 97962984.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.7448502779006958, "sampling/importance_sampling_ratio/mean": 0.3637011647224426, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.071805238723755, "sampling/sampling_logp_difference/mean": 0.011823215521872044, "step": 925, "step_time": 75.31835945998318 }, { "clip_ratio/high_max": 0.018488455330953002, "clip_ratio/high_mean": 0.007033787085674703, "clip_ratio/low_mean": 0.002694420065381564, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009728207136504352, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 844.0, "completions/mean_length": 679.875, "completions/mean_terminated_length": 532.3928833007812, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.09928167052567005, "epoch": 0.5267349260523322, "frac_reward_zero_std": 0.0, "grad_norm": 0.03265932947397232, "kl": 0.02059253735933453, "learning_rate": 4.264505119453925e-06, "loss": -0.0694, "num_tokens": 98045707.0, "reward": 1.225000023841858, "reward_std": 0.8911962509155273, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.2935006618499756, "sampling/importance_sampling_ratio/mean": 0.32480040192604065, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9827626943588257, "sampling/sampling_logp_difference/mean": 0.014220557175576687, "step": 926, "step_time": 68.38194230897352 }, { "clip_ratio/high_max": 0.018998850136995316, "clip_ratio/high_mean": 0.005917227535974234, "clip_ratio/low_mean": 0.00358405647057225, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.00950128398835659, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 930.0, "completions/mean_length": 807.9249877929688, "completions/mean_terminated_length": 591.8500366210938, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.12262251693755388, "epoch": 0.5273037542662116, "frac_reward_zero_std": 0.0, "grad_norm": 0.05894804745912552, "kl": 0.013453094521537423, "learning_rate": 4.2593856655290104e-06, "loss": -0.0052, "num_tokens": 98140808.0, "reward": 0.8500000238418579, "reward_std": 0.9212588667869568, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.4769389629364014, "sampling/importance_sampling_ratio/mean": 0.445378839969635, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4599690437316895, "sampling/sampling_logp_difference/mean": 0.014766252599656582, "step": 927, "step_time": 70.35478420113213 }, { "clip_ratio/high_max": 0.006541529612150043, "clip_ratio/high_mean": 0.0016233041969826445, "clip_ratio/low_mean": 0.0005615234440483619, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021848276155651547, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 945.7999877929688, "completions/mean_terminated_length": 633.0, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.11755532585084438, "epoch": 0.5278725824800911, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01198025792837143, "kl": 0.010703246691264212, "learning_rate": 4.254266211604096e-06, "loss": 0.0067, "num_tokens": 98257920.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.7219882011413574, "sampling/importance_sampling_ratio/mean": 0.381342351436615, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.026834487915039, "sampling/sampling_logp_difference/mean": 0.013683821074664593, "step": 928, "step_time": 77.05557982367463 }, { "clip_ratio/high_max": 0.0026705769123509526, "clip_ratio/high_mean": 0.0005341153882909566, "clip_ratio/low_mean": 0.001220703115905053, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017548184914630838, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 843.7750244140625, "completions/mean_terminated_length": 623.5, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.09179673437029123, "epoch": 0.5284414106939704, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.026647597551345825, "kl": 0.01449490780942142, "learning_rate": 4.249146757679181e-06, "loss": -0.0336, "num_tokens": 98356047.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.516534447669983, "sampling/importance_sampling_ratio/mean": 0.3156143128871918, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.367206573486328, "sampling/sampling_logp_difference/mean": 0.01337467785924673, "step": 929, "step_time": 70.84826750284992 }, { "clip_ratio/high_max": 0.010320156288798898, "clip_ratio/high_mean": 0.002936983175459318, "clip_ratio/low_mean": 0.0023437500203726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005280732992105186, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 972.125, "completions/mean_terminated_length": 835.3636474609375, "completions/min_length": 568.0, "completions/min_terminated_length": 568.0, "entropy": 0.1286014411598444, "epoch": 0.5290102389078498, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020346473902463913, "kl": 0.014751337352208793, "learning_rate": 4.244027303754266e-06, "loss": 0.0712, "num_tokens": 98472348.0, "reward": 0.550000011920929, "reward_std": 0.9044051170349121, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.7181236743927, "sampling/importance_sampling_ratio/mean": 0.31233125925064087, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.932154655456543, "sampling/sampling_logp_difference/mean": 0.014510012231767178, "step": 930, "step_time": 76.84011074877344 }, { "clip_ratio/high_max": 0.007115350803360343, "clip_ratio/high_mean": 0.002067118970444426, "clip_ratio/low_mean": 0.0034283887362107635, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005495507590239868, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 908.0, "completions/mean_length": 856.1000366210938, "completions/mean_terminated_length": 507.3846435546875, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.12071149703115225, "epoch": 0.5295790671217292, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04148270562291145, "kl": 0.013384245510678738, "learning_rate": 4.238907849829352e-06, "loss": -0.0403, "num_tokens": 98581072.0, "reward": 0.5250000357627869, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.714711904525757, "sampling/importance_sampling_ratio/mean": 0.4749719202518463, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3552358150482178, "sampling/sampling_logp_difference/mean": 0.014061209745705128, "step": 931, "step_time": 71.8543551727198 }, { "clip_ratio/high_max": 0.01533227413892746, "clip_ratio/high_mean": 0.00631401824648492, "clip_ratio/low_mean": 0.0024169922107830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008731010311748832, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 817.875, "completions/mean_terminated_length": 590.0526123046875, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.11315996758639812, "epoch": 0.5301478953356087, "frac_reward_zero_std": 0.0, "grad_norm": 0.02842877246439457, "kl": 0.014030959515366703, "learning_rate": 4.233788395904437e-06, "loss": -0.0418, "num_tokens": 98686419.0, "reward": 0.824999988079071, "reward_std": 0.9306049346923828, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.9840755462646484, "sampling/importance_sampling_ratio/mean": 0.5041893124580383, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.046476125717163, "sampling/sampling_logp_difference/mean": 0.014492476359009743, "step": 932, "step_time": 74.3076121895574 }, { "clip_ratio/high_max": 0.013699135219212621, "clip_ratio/high_mean": 0.005231713046669029, "clip_ratio/low_mean": 0.0009707251319923671, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006202438205946237, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 793.5, "completions/mean_terminated_length": 604.9091186523438, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.07448657881468534, "epoch": 0.5307167235494881, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03113044798374176, "kl": 0.010231664055027068, "learning_rate": 4.228668941979522e-06, "loss": -0.0743, "num_tokens": 98787007.0, "reward": 0.9750000238418579, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.5756843090057373, "sampling/importance_sampling_ratio/mean": 0.4302716851234436, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.197785377502441, "sampling/sampling_logp_difference/mean": 0.011027995496988297, "step": 933, "step_time": 72.55817550071515 }, { "clip_ratio/high_max": 0.015867517213337123, "clip_ratio/high_mean": 0.004164830636000261, "clip_ratio/low_mean": 0.002148437488358468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006313268095254898, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 852.0, "completions/mean_length": 894.7999877929688, "completions/mean_terminated_length": 554.1818237304688, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.12577493861317635, "epoch": 0.5312855517633674, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.006474538240581751, "kl": 0.016457128571346402, "learning_rate": 4.223549488054608e-06, "loss": 0.0001, "num_tokens": 98904391.0, "reward": 0.4000000059604645, "reward_std": 0.7089175581932068, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 0.8556271195411682, "sampling/importance_sampling_ratio/mean": 0.16155779361724854, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.472407579421997, "sampling/sampling_logp_difference/mean": 0.014520413242280483, "step": 934, "step_time": 78.13234412204474 }, { "clip_ratio/high_max": 0.010945169313345104, "clip_ratio/high_mean": 0.0029040040099062026, "clip_ratio/low_mean": 0.002350938717427198, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005254942720057443, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 902.0, "completions/mean_length": 806.9500122070312, "completions/mean_terminated_length": 646.521728515625, "completions/min_length": 401.0, "completions/min_terminated_length": 401.0, "entropy": 0.10478897672146559, "epoch": 0.5318543799772468, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.037753455340862274, "kl": 0.01678540208376944, "learning_rate": 4.218430034129693e-06, "loss": 0.068, "num_tokens": 99007173.0, "reward": 1.0, "reward_std": 0.9336995482444763, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.4785971641540527, "sampling/importance_sampling_ratio/mean": 0.4880131781101227, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4084033966064453, "sampling/sampling_logp_difference/mean": 0.014320151880383492, "step": 935, "step_time": 74.38910431601107 }, { "clip_ratio/high_max": 0.01745442015817389, "clip_ratio/high_mean": 0.006443698686780408, "clip_ratio/low_mean": 0.0016845703285071068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008128269051667303, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 932.0, "completions/mean_length": 734.6000366210938, "completions/mean_terminated_length": 472.76190185546875, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.07961615361273289, "epoch": 0.5324232081911263, "frac_reward_zero_std": 0.0, "grad_norm": 0.032097045332193375, "kl": 0.016549551161006093, "learning_rate": 4.2133105802047785e-06, "loss": -0.0765, "num_tokens": 99101277.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.298032283782959, "sampling/importance_sampling_ratio/mean": 0.4577818810939789, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6349730491638184, "sampling/sampling_logp_difference/mean": 0.011939063668251038, "step": 936, "step_time": 70.93940068036318 }, { "clip_ratio/high_max": 0.008201301330700517, "clip_ratio/high_mean": 0.0016402602923335508, "clip_ratio/low_mean": 0.0020263672049622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003666627424536273, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 523.0, "completions/mean_length": 923.5250244140625, "completions/mean_terminated_length": 354.16668701171875, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.05950137972831726, "epoch": 0.5329920364050057, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.047149911522865295, "kl": 0.014412180811632425, "learning_rate": 4.208191126279864e-06, "loss": -0.0613, "num_tokens": 99195466.0, "reward": 0.30000001192092896, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.187971830368042, "sampling/importance_sampling_ratio/mean": 0.3268604278564453, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.302966594696045, "sampling/sampling_logp_difference/mean": 0.008815395645797253, "step": 937, "step_time": 70.78992701740935 }, { "clip_ratio/high_max": 0.014636034960858524, "clip_ratio/high_mean": 0.005219586819293909, "clip_ratio/low_mean": 0.0022802734238211997, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007499860250391066, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 823.7999877929688, "completions/mean_terminated_length": 552.941162109375, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.11701840069144964, "epoch": 0.5335608646188851, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01949954405426979, "kl": 0.012247306178323925, "learning_rate": 4.2030716723549485e-06, "loss": -0.0088, "num_tokens": 99288866.0, "reward": 0.574999988079071, "reward_std": 0.780778706073761, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.4574480056762695, "sampling/importance_sampling_ratio/mean": 0.356999933719635, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.461498737335205, "sampling/sampling_logp_difference/mean": 0.013981749303638935, "step": 938, "step_time": 72.10267818672583 }, { "clip_ratio/high_max": 0.015655007446184754, "clip_ratio/high_mean": 0.004735862836241722, "clip_ratio/low_mean": 0.002059788806946017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006795651512220502, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 741.6749877929688, "completions/mean_terminated_length": 605.74072265625, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.091956433840096, "epoch": 0.5341296928327645, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.034675437957048416, "kl": 0.01944109727628529, "learning_rate": 4.197952218430035e-06, "loss": 0.0346, "num_tokens": 99392381.0, "reward": 1.225000023841858, "reward_std": 0.9195177555084229, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.002627372741699, "sampling/importance_sampling_ratio/mean": 0.42957931756973267, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0736141204833984, "sampling/sampling_logp_difference/mean": 0.013039357028901577, "step": 939, "step_time": 70.66482559079304 }, { "clip_ratio/high_max": 0.0057438507792539895, "clip_ratio/high_mean": 0.0017260907916352153, "clip_ratio/low_mean": 0.0021240234636934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038501142989844084, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 854.7250366210938, "completions/mean_terminated_length": 600.8125, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.10733398050069809, "epoch": 0.534698521046644, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014989566057920456, "kl": 0.012251294450834394, "learning_rate": 4.192832764505119e-06, "loss": 0.0344, "num_tokens": 99501754.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.9827637672424316, "sampling/importance_sampling_ratio/mean": 0.6341614127159119, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2223246097564697, "sampling/sampling_logp_difference/mean": 0.013687456026673317, "step": 940, "step_time": 73.22797593660653 }, { "clip_ratio/high_max": 0.012518560513854027, "clip_ratio/high_mean": 0.003076152235735208, "clip_ratio/low_mean": 0.001904296896100277, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004980449069989845, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 959.0, "completions/mean_length": 812.2000122070312, "completions/mean_terminated_length": 494.5, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.12953697796911, "epoch": 0.5352673492605233, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02940649725496769, "kl": 0.012844466837123036, "learning_rate": 4.187713310580205e-06, "loss": -0.1032, "num_tokens": 99609186.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.1605324745178223, "sampling/importance_sampling_ratio/mean": 0.29402956366539, "sampling/importance_sampling_ratio/min": 0.0028777841944247484, "sampling/sampling_logp_difference/max": 1.9650179147720337, "sampling/sampling_logp_difference/mean": 0.015879249200224876, "step": 941, "step_time": 73.07597257732414 }, { "clip_ratio/high_max": 0.009488004201557487, "clip_ratio/high_mean": 0.002477325891959481, "clip_ratio/low_mean": 0.0030750785081181675, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005552404443733394, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 834.5250244140625, "completions/mean_terminated_length": 645.0499877929688, "completions/min_length": 417.0, "completions/min_terminated_length": 417.0, "entropy": 0.1598817091435194, "epoch": 0.5358361774744027, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013286442495882511, "kl": 0.014691674499772489, "learning_rate": 4.18259385665529e-06, "loss": 0.042, "num_tokens": 99724295.0, "reward": 0.875, "reward_std": 0.9388344883918762, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.6076664924621582, "sampling/importance_sampling_ratio/mean": 0.24992862343788147, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9742001295089722, "sampling/sampling_logp_difference/mean": 0.017707064747810364, "step": 942, "step_time": 76.74471556348726 }, { "clip_ratio/high_max": 0.012911855126731098, "clip_ratio/high_mean": 0.0047613194765290245, "clip_ratio/low_mean": 0.0017089843877329258, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006470303749665618, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 805.0250244140625, "completions/mean_terminated_length": 606.90478515625, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.10009316448122263, "epoch": 0.5364050056882821, "frac_reward_zero_std": 0.0, "grad_norm": 0.02921966277062893, "kl": 0.014319326844997704, "learning_rate": 4.177474402730376e-06, "loss": -0.1471, "num_tokens": 99832256.0, "reward": 0.949999988079071, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.379518985748291, "sampling/importance_sampling_ratio/mean": 0.37462979555130005, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.215853214263916, "sampling/sampling_logp_difference/mean": 0.01259764563292265, "step": 943, "step_time": 76.98228449327871 }, { "clip_ratio/high_max": 0.008261145267169923, "clip_ratio/high_mean": 0.0018734679761109874, "clip_ratio/low_mean": 0.001635742202779511, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035092101898044348, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 841.1000366210938, "completions/mean_terminated_length": 536.2667236328125, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.08624918758869171, "epoch": 0.5369738339021616, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016716821119189262, "kl": 0.00868149078451097, "learning_rate": 4.172354948805461e-06, "loss": 0.0686, "num_tokens": 99934108.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.8180854320526123, "sampling/importance_sampling_ratio/mean": 0.44391077756881714, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9509005546569824, "sampling/sampling_logp_difference/mean": 0.011632973328232765, "step": 944, "step_time": 71.01856312714517 }, { "clip_ratio/high_max": 0.013131003710441291, "clip_ratio/high_mean": 0.004768590268213302, "clip_ratio/low_mean": 0.002099609380820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0068681996781378984, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 881.7999877929688, "completions/mean_terminated_length": 708.0, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "entropy": 0.11988616175949574, "epoch": 0.537542662116041, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014815493486821651, "kl": 0.009984075906686485, "learning_rate": 4.1672354948805465e-06, "loss": 0.0567, "num_tokens": 100022716.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.5782523155212402, "sampling/importance_sampling_ratio/mean": 0.3904626965522766, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8283354043960571, "sampling/sampling_logp_difference/mean": 0.013513711281120777, "step": 945, "step_time": 70.78427529591136 }, { "clip_ratio/high_max": 0.007780554587952793, "clip_ratio/high_mean": 0.002609011484310031, "clip_ratio/low_mean": 0.001953125036379788, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004562136513413861, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 906.25, "completions/mean_terminated_length": 661.6923217773438, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.10759821441024542, "epoch": 0.5381114903299203, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03325051814317703, "kl": 0.011060199583880603, "learning_rate": 4.162116040955631e-06, "loss": -0.0344, "num_tokens": 100130158.0, "reward": 0.5, "reward_std": 0.7844645380973816, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.5026609897613525, "sampling/importance_sampling_ratio/mean": 0.4026148319244385, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2226483821868896, "sampling/sampling_logp_difference/mean": 0.012640866450965405, "step": 946, "step_time": 72.05189915676601 }, { "clip_ratio/high_max": 0.014140024897642434, "clip_ratio/high_mean": 0.004794679276528768, "clip_ratio/low_mean": 0.0022559804128832184, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007050659507513046, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 968.0, "completions/mean_length": 867.6749877929688, "completions/mean_terminated_length": 656.1764526367188, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.1118017053231597, "epoch": 0.5386803185437997, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.031260013580322266, "kl": 0.017171045183204114, "learning_rate": 4.156996587030717e-06, "loss": -0.1512, "num_tokens": 100243801.0, "reward": 0.7750000357627869, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.6208391189575195, "sampling/importance_sampling_ratio/mean": 0.3135724663734436, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3138158321380615, "sampling/sampling_logp_difference/mean": 0.01477679330855608, "step": 947, "step_time": 75.78287792857736 }, { "clip_ratio/high_max": 0.013048436492681503, "clip_ratio/high_mean": 0.004300185391912237, "clip_ratio/low_mean": 0.001733398443320766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006033583777025342, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 963.0, "completions/mean_length": 879.8250122070312, "completions/mean_terminated_length": 639.5333862304688, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.11823876202106476, "epoch": 0.5392491467576792, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01768556609749794, "kl": 0.013260673033073545, "learning_rate": 4.151877133105802e-06, "loss": -0.0252, "num_tokens": 100355346.0, "reward": 0.699999988079071, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.5965490341186523, "sampling/importance_sampling_ratio/mean": 0.43941158056259155, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.074594020843506, "sampling/sampling_logp_difference/mean": 0.013445849530398846, "step": 948, "step_time": 75.47564836917445 }, { "clip_ratio/high_max": 0.012848351267166436, "clip_ratio/high_mean": 0.003824862855253741, "clip_ratio/low_mean": 0.003203919932275312, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0070287828566506505, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 893.0, "completions/mean_length": 807.1749877929688, "completions/mean_terminated_length": 590.3500366210938, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.08654578868299723, "epoch": 0.5398179749715586, "frac_reward_zero_std": 0.0, "grad_norm": 0.049095045775175095, "kl": 0.014850138803012669, "learning_rate": 4.1467576791808874e-06, "loss": -0.1458, "num_tokens": 100455449.0, "reward": 0.7750000357627869, "reward_std": 0.8619446754455566, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.6945722103118896, "sampling/importance_sampling_ratio/mean": 0.286080926656723, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.221222400665283, "sampling/sampling_logp_difference/mean": 0.013346520252525806, "step": 949, "step_time": 71.55985599127598 }, { "clip_ratio/high_max": 0.015468627505470067, "clip_ratio/high_mean": 0.004813580366317183, "clip_ratio/low_mean": 0.0015136718429857865, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006327252252958715, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 904.0, "completions/mean_length": 874.4500122070312, "completions/mean_terminated_length": 650.125, "completions/min_length": 387.0, "completions/min_terminated_length": 387.0, "entropy": 0.0836697481572628, "epoch": 0.540386803185438, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.009936423972249031, "kl": 0.014313757070340216, "learning_rate": 4.141638225255973e-06, "loss": 0.013, "num_tokens": 100566979.0, "reward": 0.675000011920929, "reward_std": 0.888314425945282, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.6857523918151855, "sampling/importance_sampling_ratio/mean": 0.30529123544692993, "sampling/importance_sampling_ratio/min": 0.002782551571726799, "sampling/sampling_logp_difference/max": 3.2134616374969482, "sampling/sampling_logp_difference/mean": 0.011459404602646828, "step": 950, "step_time": 75.56255490262993 }, { "clip_ratio/high_max": 0.009116200846619904, "clip_ratio/high_mean": 0.0021550935562117957, "clip_ratio/low_mean": 0.0012451172078726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034002107568085194, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 891.0, "completions/mean_length": 830.8500366210938, "completions/mean_terminated_length": 569.5294189453125, "completions/min_length": 385.0, "completions/min_terminated_length": 385.0, "entropy": 0.0750323198735714, "epoch": 0.5409556313993175, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019439997151494026, "kl": 0.009480367007199675, "learning_rate": 4.136518771331058e-06, "loss": -0.014, "num_tokens": 100667781.0, "reward": 0.6500000357627869, "reward_std": 0.8335896730422974, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.323890209197998, "sampling/importance_sampling_ratio/mean": 0.4117274284362793, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4769632816314697, "sampling/sampling_logp_difference/mean": 0.010280056856572628, "step": 951, "step_time": 71.26185599993914 }, { "clip_ratio/high_max": 0.0054261754266917706, "clip_ratio/high_mean": 0.001553457579575479, "clip_ratio/low_mean": 0.002329792932869168, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038832504651509225, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 688.0, "completions/mean_length": 701.4000244140625, "completions/mean_terminated_length": 486.3333435058594, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.10930867027491331, "epoch": 0.5415244596131968, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04157298058271408, "kl": 0.013934826594777405, "learning_rate": 4.131399317406144e-06, "loss": -0.0767, "num_tokens": 100765317.0, "reward": 1.149999976158142, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.5491299629211426, "sampling/importance_sampling_ratio/mean": 0.5002330541610718, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3160042762756348, "sampling/sampling_logp_difference/mean": 0.015203529968857765, "step": 952, "step_time": 70.43926946306601 }, { "clip_ratio/high_max": 0.014027648139744997, "clip_ratio/high_mean": 0.0034374057431705296, "clip_ratio/low_mean": 0.0012939453172293724, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004731351160444319, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 739.9500122070312, "completions/mean_terminated_length": 603.1851806640625, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.10816649068146944, "epoch": 0.5420932878270762, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.031590621918439865, "kl": 0.019508584518916905, "learning_rate": 4.126279863481228e-06, "loss": 0.1426, "num_tokens": 100868451.0, "reward": 1.350000023841858, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.7784953117370605, "sampling/importance_sampling_ratio/mean": 0.32704898715019226, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.9977192878723145, "sampling/sampling_logp_difference/mean": 0.01488791685551405, "step": 953, "step_time": 73.74428351246752 }, { "clip_ratio/high_max": 0.011050412809709087, "clip_ratio/high_mean": 0.0041436295650782995, "clip_ratio/low_mean": 0.0019042968924622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006047926377505064, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 894.8250122070312, "completions/mean_terminated_length": 752.0526123046875, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "entropy": 0.12112319003790617, "epoch": 0.5426621160409556, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01897238940000534, "kl": 0.015510202967561781, "learning_rate": 4.121160409556314e-06, "loss": 0.0307, "num_tokens": 100975932.0, "reward": 0.824999988079071, "reward_std": 0.9306049942970276, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.536045789718628, "sampling/importance_sampling_ratio/mean": 0.28115153312683105, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0278193950653076, "sampling/sampling_logp_difference/mean": 0.014860830269753933, "step": 954, "step_time": 78.10432453430258 }, { "clip_ratio/high_max": 0.004630932700820267, "clip_ratio/high_mean": 0.0011593954841373488, "clip_ratio/low_mean": 0.002416992205326096, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003576387622160837, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 873.9500122070312, "completions/mean_terminated_length": 562.3077392578125, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.1088439617305994, "epoch": 0.5432309442548351, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.034634578973054886, "kl": 0.013969851424917579, "learning_rate": 4.116040955631399e-06, "loss": -0.0739, "num_tokens": 101078474.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.6333248615264893, "sampling/importance_sampling_ratio/mean": 0.28295034170150757, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4557652473449707, "sampling/sampling_logp_difference/mean": 0.01323768775910139, "step": 955, "step_time": 73.86264463188127 }, { "clip_ratio/high_max": 0.012769433786161244, "clip_ratio/high_mean": 0.0032485670235473663, "clip_ratio/low_mean": 0.0007812500080035534, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004029817093396559, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 934.0, "completions/mean_length": 655.6500244140625, "completions/mean_terminated_length": 497.7857360839844, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.09385374002158642, "epoch": 0.5437997724687145, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03308309614658356, "kl": 0.017298903316259384, "learning_rate": 4.110921501706485e-06, "loss": -0.0281, "num_tokens": 101178764.0, "reward": 1.375, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.8539059162139893, "sampling/importance_sampling_ratio/mean": 0.5904325246810913, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7619261741638184, "sampling/sampling_logp_difference/mean": 0.013479155488312244, "step": 956, "step_time": 70.48653323738836 }, { "clip_ratio/high_max": 0.01613671926315874, "clip_ratio/high_mean": 0.005505774810444564, "clip_ratio/low_mean": 0.0039950995342223905, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.009500874439254403, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 813.2000122070312, "completions/mean_terminated_length": 622.4761962890625, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.1623500371351838, "epoch": 0.5443686006825939, "frac_reward_zero_std": 0.0, "grad_norm": 0.04783828556537628, "kl": 0.017312597250565886, "learning_rate": 4.10580204778157e-06, "loss": 0.0043, "num_tokens": 101289260.0, "reward": 0.800000011920929, "reward_std": 0.8533493876457214, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.6772429943084717, "sampling/importance_sampling_ratio/mean": 0.3729512095451355, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9110198020935059, "sampling/sampling_logp_difference/mean": 0.01824762672185898, "step": 957, "step_time": 75.71556505700573 }, { "clip_ratio/high_max": 0.012668361538089812, "clip_ratio/high_mean": 0.00544453234761022, "clip_ratio/low_mean": 0.0014160156388243195, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00686054810648784, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 801.5499877929688, "completions/mean_terminated_length": 600.2857055664062, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.08394285850226879, "epoch": 0.5449374288964732, "frac_reward_zero_std": 0.0, "grad_norm": 0.026372618973255157, "kl": 0.01246973080560565, "learning_rate": 4.1006825938566555e-06, "loss": -0.0611, "num_tokens": 101385434.0, "reward": 1.024999976158142, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.1780362129211426, "sampling/importance_sampling_ratio/mean": 0.3887253701686859, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.984004259109497, "sampling/sampling_logp_difference/mean": 0.01197583694010973, "step": 958, "step_time": 69.38244592002593 }, { "clip_ratio/high_max": 0.01636387046892196, "clip_ratio/high_mean": 0.00579847177141346, "clip_ratio/low_mean": 0.004301590684917755, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01010006241267547, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 782.625, "completions/mean_terminated_length": 621.7083740234375, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.17497462034225464, "epoch": 0.5455062571103527, "frac_reward_zero_std": 0.0, "grad_norm": 0.022907273843884468, "kl": 0.0179977600928396, "learning_rate": 4.095563139931741e-06, "loss": 0.0592, "num_tokens": 101492227.0, "reward": 0.9000000357627869, "reward_std": 0.8412445187568665, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.0299947261810303, "sampling/importance_sampling_ratio/mean": 0.3260267972946167, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5586843490600586, "sampling/sampling_logp_difference/mean": 0.01944856159389019, "step": 959, "step_time": 74.41110037895851 }, { "clip_ratio/high_max": 0.011680530151352286, "clip_ratio/high_mean": 0.00334703332191566, "clip_ratio/low_mean": 0.002661132806679234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006008165946695954, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 822.6749877929688, "completions/mean_terminated_length": 600.1578979492188, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.13190342020243406, "epoch": 0.5460750853242321, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02526809088885784, "kl": 0.013847091817297041, "learning_rate": 4.090443686006826e-06, "loss": -0.0382, "num_tokens": 101603270.0, "reward": 0.8500000238418579, "reward_std": 0.9486833214759827, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.916003942489624, "sampling/importance_sampling_ratio/mean": 0.4335477948188782, "sampling/importance_sampling_ratio/min": 0.0011052001500502229, "sampling/sampling_logp_difference/max": 3.631279230117798, "sampling/sampling_logp_difference/mean": 0.015815194696187973, "step": 960, "step_time": 76.65376833337359 }, { "clip_ratio/high_max": 0.016811618930660188, "clip_ratio/high_mean": 0.0068465521617326885, "clip_ratio/low_mean": 0.0009964293167286087, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007842981431167573, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 885.0, "completions/mean_length": 691.0750122070312, "completions/mean_terminated_length": 607.84375, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.11445281840860844, "epoch": 0.5466439135381115, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014782574959099293, "kl": 0.017491544713266194, "learning_rate": 4.085324232081911e-06, "loss": -0.0736, "num_tokens": 101692249.0, "reward": 1.375, "reward_std": 0.8066232204437256, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.658060073852539, "sampling/importance_sampling_ratio/mean": 0.35925379395484924, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6568119525909424, "sampling/sampling_logp_difference/mean": 0.015124186873435974, "step": 961, "step_time": 71.15573768224567 }, { "clip_ratio/high_max": 0.010235819034278393, "clip_ratio/high_mean": 0.002648700843565166, "clip_ratio/low_mean": 0.0024202341737691313, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005068935017334297, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 805.0750122070312, "completions/mean_terminated_length": 687.1923217773438, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.08540353458374739, "epoch": 0.5472127417519909, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023782260715961456, "kl": 0.01080203236779198, "learning_rate": 4.080204778156996e-06, "loss": 0.1091, "num_tokens": 101779428.0, "reward": 1.0500000715255737, "reward_std": 0.875594973564148, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.494408130645752, "sampling/importance_sampling_ratio/mean": 0.4845430254936218, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7075977325439453, "sampling/sampling_logp_difference/mean": 0.01212582178413868, "step": 962, "step_time": 73.97092323866673 }, { "clip_ratio/high_max": 0.014129902876447886, "clip_ratio/high_mean": 0.0029004592215642333, "clip_ratio/low_mean": 0.00190429688700533, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004804756143130362, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 842.2999877929688, "completions/mean_terminated_length": 596.4705810546875, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.10410331003367901, "epoch": 0.5477815699658704, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.005509000271558762, "kl": 0.011997801600955427, "learning_rate": 4.075085324232082e-06, "loss": 0.0218, "num_tokens": 101886168.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.8757784366607666, "sampling/importance_sampling_ratio/mean": 0.24909770488739014, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.21868634223938, "sampling/sampling_logp_difference/mean": 0.012952491641044617, "step": 963, "step_time": 72.45448168902658 }, { "clip_ratio/high_max": 0.015761847142130136, "clip_ratio/high_mean": 0.005683826020685956, "clip_ratio/low_mean": 0.0021542503527598456, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007838076213374734, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 977.0, "completions/mean_length": 805.0250244140625, "completions/mean_terminated_length": 563.0, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.12627539224922657, "epoch": 0.5483503981797497, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.036604661494493484, "kl": 0.014580085058696568, "learning_rate": 4.069965870307167e-06, "loss": -0.1161, "num_tokens": 101986729.0, "reward": 0.800000011920929, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.5674517154693604, "sampling/importance_sampling_ratio/mean": 0.5180812478065491, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7931737899780273, "sampling/sampling_logp_difference/mean": 0.015028297901153564, "step": 964, "step_time": 73.2607243922539 }, { "clip_ratio/high_max": 0.006348066497594118, "clip_ratio/high_mean": 0.0014832885062787682, "clip_ratio/low_mean": 0.0017822265508584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032655150280334055, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 855.3250122070312, "completions/mean_terminated_length": 542.0714721679688, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.06758613977581263, "epoch": 0.5489192263936291, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.012716235592961311, "kl": 0.009786730515770614, "learning_rate": 4.064846416382253e-06, "loss": 0.0129, "num_tokens": 102097974.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.4663070440292358, "sampling/importance_sampling_ratio/mean": 0.30297645926475525, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.81057071685791, "sampling/sampling_logp_difference/mean": 0.01018446683883667, "step": 965, "step_time": 74.39255685522221 }, { "clip_ratio/high_max": 0.010038114036433399, "clip_ratio/high_mean": 0.0037784147425554693, "clip_ratio/low_mean": 0.002075195321594947, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.005853610171470791, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 944.0, "completions/mean_length": 848.9750366210938, "completions/mean_terminated_length": 586.4375, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.054902114905416965, "epoch": 0.5494880546075085, "frac_reward_zero_std": 0.0, "grad_norm": 0.028298018500208855, "kl": 0.010030352277681231, "learning_rate": 4.059726962457338e-06, "loss": -0.0986, "num_tokens": 102204685.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.3084757328033447, "sampling/importance_sampling_ratio/mean": 0.4682559370994568, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1078972816467285, "sampling/sampling_logp_difference/mean": 0.008613242767751217, "step": 966, "step_time": 74.20548398699611 }, { "clip_ratio/high_max": 0.01522872771602124, "clip_ratio/high_mean": 0.007239962927997112, "clip_ratio/low_mean": 0.00219718230800936, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009437145257834345, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 851.0, "completions/mean_length": 637.1500244140625, "completions/mean_terminated_length": 524.8386840820312, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.09595744777470827, "epoch": 0.550056882821388, "frac_reward_zero_std": 0.0, "grad_norm": 0.043842218816280365, "kl": 0.015205944189801812, "learning_rate": 4.0546075085324236e-06, "loss": -0.0961, "num_tokens": 102307523.0, "reward": 1.149999976158142, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.904995918273926, "sampling/importance_sampling_ratio/mean": 0.5958130955696106, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.881053924560547, "sampling/sampling_logp_difference/mean": 0.01428955513983965, "step": 967, "step_time": 71.96663381904364 }, { "clip_ratio/high_max": 0.011520104948431253, "clip_ratio/high_mean": 0.00379579360014759, "clip_ratio/low_mean": 0.001245999133971054, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005041792639531195, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 780.2250366210938, "completions/mean_terminated_length": 450.4117736816406, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.06803274294361472, "epoch": 0.5506257110352674, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018387775868177414, "kl": 0.011092737200669944, "learning_rate": 4.049488054607509e-06, "loss": 0.0376, "num_tokens": 102407788.0, "reward": 0.7250000238418579, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.4544897079467773, "sampling/importance_sampling_ratio/mean": 0.3817582428455353, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.6436500549316406, "sampling/sampling_logp_difference/mean": 0.010741367004811764, "step": 968, "step_time": 70.98764427774586 }, { "clip_ratio/high_max": 0.011858671088702977, "clip_ratio/high_mean": 0.0038897148333489895, "clip_ratio/low_mean": 0.0017896414101414848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00567935622530058, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 786.9249877929688, "completions/mean_terminated_length": 659.2692260742188, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 0.0940558323636651, "epoch": 0.5511945392491467, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013605175539851189, "kl": 0.012682953965850174, "learning_rate": 4.0443686006825936e-06, "loss": 0.0105, "num_tokens": 102501201.0, "reward": 1.274999976158142, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.7981663942337036, "sampling/importance_sampling_ratio/mean": 0.3092889189720154, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9859843254089355, "sampling/sampling_logp_difference/mean": 0.013293430209159851, "step": 969, "step_time": 71.69849870540202 }, { "clip_ratio/high_max": 0.006831991020590067, "clip_ratio/high_mean": 0.0016000430914573371, "clip_ratio/low_mean": 0.0027377457881812006, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004337788792327046, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 968.0, "completions/mean_length": 850.9000244140625, "completions/mean_terminated_length": 562.4000244140625, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.13500104285776615, "epoch": 0.5517633674630261, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018086588010191917, "kl": 0.011838724371045828, "learning_rate": 4.039249146757679e-06, "loss": 0.0145, "num_tokens": 102613837.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.3586227893829346, "sampling/importance_sampling_ratio/mean": 0.5261691808700562, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2360355854034424, "sampling/sampling_logp_difference/mean": 0.01606404036283493, "step": 970, "step_time": 73.4343154290691 }, { "clip_ratio/high_max": 0.012770178145729005, "clip_ratio/high_mean": 0.0044986914144828916, "clip_ratio/low_mean": 0.002450149448122829, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006948840746190399, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 967.0, "completions/mean_length": 833.25, "completions/mean_terminated_length": 642.5, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "entropy": 0.09895874839276075, "epoch": 0.5523321956769056, "frac_reward_zero_std": 0.0, "grad_norm": 0.04084331542253494, "kl": 0.013171252445317805, "learning_rate": 4.0341296928327644e-06, "loss": -0.1334, "num_tokens": 102721399.0, "reward": 0.925000011920929, "reward_std": 0.9710555076599121, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.8413102626800537, "sampling/importance_sampling_ratio/mean": 0.4370715022087097, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8346376419067383, "sampling/sampling_logp_difference/mean": 0.012840303592383862, "step": 971, "step_time": 74.70485809468664 }, { "clip_ratio/high_max": 0.010393462551292032, "clip_ratio/high_mean": 0.0026365265948697925, "clip_ratio/low_mean": 0.0012087938812328503, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038453204033430666, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 670.1749877929688, "completions/mean_terminated_length": 499.8148193359375, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.11100140307098627, "epoch": 0.552901023890785, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.027804670855402946, "kl": 0.01644565467722714, "learning_rate": 4.02901023890785e-06, "loss": 0.0148, "num_tokens": 102823966.0, "reward": 1.3000000715255737, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.6859731674194336, "sampling/importance_sampling_ratio/mean": 0.49004918336868286, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.296351909637451, "sampling/sampling_logp_difference/mean": 0.016722222790122032, "step": 972, "step_time": 72.10517147439532 }, { "clip_ratio/high_max": 0.016591989318840206, "clip_ratio/high_mean": 0.006170784181449562, "clip_ratio/low_mean": 0.0013797625033475924, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0075505466083996, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 713.7250366210938, "completions/mean_terminated_length": 564.3333129882812, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.09757953602820635, "epoch": 0.5534698521046644, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024086304008960724, "kl": 0.015004066284745932, "learning_rate": 4.023890784982935e-06, "loss": -0.0635, "num_tokens": 102916627.0, "reward": 1.225000023841858, "reward_std": 0.9195176959037781, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9876527786254883, "sampling/importance_sampling_ratio/mean": 0.5997629761695862, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.890127420425415, "sampling/sampling_logp_difference/mean": 0.014147465117275715, "step": 973, "step_time": 73.62562075583264 }, { "clip_ratio/high_max": 0.012442333274520934, "clip_ratio/high_mean": 0.004091176495421678, "clip_ratio/low_mean": 0.0029785156075377017, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.007069692015647888, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 797.0, "completions/mean_length": 863.5, "completions/mean_terminated_length": 565.4285888671875, "completions/min_length": 403.0, "completions/min_terminated_length": 403.0, "entropy": 0.11157400067895651, "epoch": 0.5540386803185438, "frac_reward_zero_std": 0.0, "grad_norm": 0.015800829976797104, "kl": 0.013790460187010467, "learning_rate": 4.018771331058021e-06, "loss": -0.0335, "num_tokens": 103028255.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 0.7152755856513977, "sampling/importance_sampling_ratio/mean": 0.2029813826084137, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.271176338195801, "sampling/sampling_logp_difference/mean": 0.014181370846927166, "step": 974, "step_time": 74.72822943585925 }, { "clip_ratio/high_max": 0.008871181518770754, "clip_ratio/high_mean": 0.0021870803684578277, "clip_ratio/low_mean": 0.002539062454161467, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0047261428262572736, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 769.9000244140625, "completions/mean_terminated_length": 562.0, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.13421162497252226, "epoch": 0.5546075085324232, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015879742801189423, "kl": 0.01566891954280436, "learning_rate": 4.013651877133106e-06, "loss": 0.0218, "num_tokens": 103128395.0, "reward": 1.0750000476837158, "reward_std": 0.9971111416816711, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.6411311626434326, "sampling/importance_sampling_ratio/mean": 0.5817481279373169, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8907839059829712, "sampling/sampling_logp_difference/mean": 0.015697820112109184, "step": 975, "step_time": 72.13724207459018 }, { "clip_ratio/high_max": 0.015086932748090476, "clip_ratio/high_mean": 0.005850610381457955, "clip_ratio/low_mean": 0.0010493888712517219, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0068999992799945176, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 822.0, "completions/mean_length": 739.0, "completions/mean_terminated_length": 505.8182067871094, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.08469514735043049, "epoch": 0.5551763367463026, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027664270251989365, "kl": 0.013968001934699714, "learning_rate": 4.008532423208192e-06, "loss": 0.0029, "num_tokens": 103231947.0, "reward": 1.0750000476837158, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.395731806755066, "sampling/importance_sampling_ratio/mean": 0.36930423974990845, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.500241279602051, "sampling/sampling_logp_difference/mean": 0.012087827548384666, "step": 976, "step_time": 72.60377149423584 }, { "clip_ratio/high_max": 0.00698285864200443, "clip_ratio/high_mean": 0.0016434853023383766, "clip_ratio/low_mean": 0.001220703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028641884127864614, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 972.3500366210938, "completions/mean_terminated_length": 610.7999877929688, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "entropy": 0.0959946746006608, "epoch": 0.555745164960182, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.03660448640584946, "kl": 0.009381362586282194, "learning_rate": 4.003412969283276e-06, "loss": -0.0725, "num_tokens": 103347713.0, "reward": 0.25, "reward_std": 0.6698641180992126, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.9206089973449707, "sampling/importance_sampling_ratio/mean": 0.4619692265987396, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.71083664894104, "sampling/sampling_logp_difference/mean": 0.011671277694404125, "step": 977, "step_time": 76.16899866098538 }, { "clip_ratio/high_max": 0.013180260313674808, "clip_ratio/high_mean": 0.003992702811956406, "clip_ratio/low_mean": 0.0028076171438442543, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.0068003201158717275, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 880.5250244140625, "completions/mean_terminated_length": 641.4000244140625, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.08107304945588112, "epoch": 0.5563139931740614, "frac_reward_zero_std": 0.0, "grad_norm": 0.03132924437522888, "kl": 0.009969462873414159, "learning_rate": 3.9982935153583625e-06, "loss": 0.053, "num_tokens": 103458486.0, "reward": 0.574999988079071, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.8195420503616333, "sampling/importance_sampling_ratio/mean": 0.46744269132614136, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.23671293258667, "sampling/sampling_logp_difference/mean": 0.01059752143919468, "step": 978, "step_time": 73.51185231818818 }, { "clip_ratio/high_max": 0.007690756407100707, "clip_ratio/high_mean": 0.002645967098942492, "clip_ratio/low_mean": 0.0015625000087311491, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0042084670858457685, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 696.0, "completions/mean_length": 641.2250366210938, "completions/mean_terminated_length": 435.1153869628906, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.08442936837673187, "epoch": 0.5568828213879409, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03394488990306854, "kl": 0.0188723336905241, "learning_rate": 3.993174061433447e-06, "loss": -0.0736, "num_tokens": 103557599.0, "reward": 1.3000000715255737, "reward_std": 0.9660917520523071, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.7377583980560303, "sampling/importance_sampling_ratio/mean": 0.5819026231765747, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.197782278060913, "sampling/sampling_logp_difference/mean": 0.013442802242934704, "step": 979, "step_time": 70.70806213049218 }, { "clip_ratio/high_max": 0.01106290757888928, "clip_ratio/high_mean": 0.0027253224689047784, "clip_ratio/low_mean": 0.0007080078248691279, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003433330261032097, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 953.0, "completions/mean_length": 839.5250244140625, "completions/mean_terminated_length": 589.941162109375, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.09806028008460999, "epoch": 0.5574516496018203, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01238386519253254, "kl": 0.015917847049422562, "learning_rate": 3.9880546075085325e-06, "loss": -0.0123, "num_tokens": 103662604.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.566236972808838, "sampling/importance_sampling_ratio/mean": 0.27402156591415405, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.459980010986328, "sampling/sampling_logp_difference/mean": 0.013880687765777111, "step": 980, "step_time": 72.5215174597688 }, { "clip_ratio/high_max": 0.01806598308030516, "clip_ratio/high_mean": 0.005190224619582295, "clip_ratio/low_mean": 0.0027285891483188607, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007918813789729029, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 888.0, "completions/mean_length": 786.1500244140625, "completions/mean_terminated_length": 523.26318359375, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.120926802046597, "epoch": 0.5580204778156996, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02510043792426586, "kl": 0.013971820357255638, "learning_rate": 3.982935153583618e-06, "loss": 0.027, "num_tokens": 103762794.0, "reward": 0.8500000238418579, "reward_std": 0.9486833214759827, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.7536356449127197, "sampling/importance_sampling_ratio/mean": 0.513518214225769, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5536189079284668, "sampling/sampling_logp_difference/mean": 0.014495055191218853, "step": 981, "step_time": 69.52837341302074 }, { "clip_ratio/high_max": 0.01156033156439662, "clip_ratio/high_mean": 0.003481151259620674, "clip_ratio/low_mean": 0.0027683124426403083, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006249463825952262, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 940.0, "completions/mean_length": 891.1500244140625, "completions/mean_terminated_length": 691.875, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.09403431508690119, "epoch": 0.558589306029579, "frac_reward_zero_std": 0.0, "grad_norm": 0.033283885568380356, "kl": 0.02600638277363032, "learning_rate": 3.977815699658703e-06, "loss": 0.0916, "num_tokens": 103863264.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.0257444381713867, "sampling/importance_sampling_ratio/mean": 0.4205693304538727, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2417211532592773, "sampling/sampling_logp_difference/mean": 0.011884679086506367, "step": 982, "step_time": 75.85876450897194 }, { "clip_ratio/high_max": 0.008408314432017505, "clip_ratio/high_mean": 0.0026050186716020107, "clip_ratio/low_mean": 0.002587890590802999, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005192909273318946, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 936.0, "completions/mean_length": 894.3500366210938, "completions/mean_terminated_length": 591.8333740234375, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "entropy": 0.09020262211561203, "epoch": 0.5591581342434585, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024521926417946815, "kl": 0.010907383169978857, "learning_rate": 3.972696245733789e-06, "loss": -0.0172, "num_tokens": 103972750.0, "reward": 0.4749999940395355, "reward_std": 0.7840558886528015, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.462566614151001, "sampling/importance_sampling_ratio/mean": 0.49847322702407837, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.124973773956299, "sampling/sampling_logp_difference/mean": 0.011969588696956635, "step": 983, "step_time": 75.31635475461371 }, { "clip_ratio/high_max": 0.018013939377851784, "clip_ratio/high_mean": 0.006744532409356907, "clip_ratio/low_mean": 0.001635742193684564, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008380274579394609, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 990.0, "completions/mean_length": 812.0250244140625, "completions/mean_terminated_length": 655.3478393554688, "completions/min_length": 403.0, "completions/min_terminated_length": 403.0, "entropy": 0.12411836348474026, "epoch": 0.5597269624573379, "frac_reward_zero_std": 0.0, "grad_norm": 0.025647923350334167, "kl": 0.012322106631472707, "learning_rate": 3.967576791808874e-06, "loss": 0.0712, "num_tokens": 104077119.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.988020420074463, "sampling/importance_sampling_ratio/mean": 0.27438220381736755, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5702147483825684, "sampling/sampling_logp_difference/mean": 0.014740916900336742, "step": 984, "step_time": 72.88041119463742 }, { "clip_ratio/high_max": 0.01795488700736314, "clip_ratio/high_mean": 0.004372182942461222, "clip_ratio/low_mean": 0.0014554440276697278, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005827626882819459, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 871.7750244140625, "completions/mean_terminated_length": 703.5263061523438, "completions/min_length": 381.0, "completions/min_terminated_length": 381.0, "entropy": 0.11288719065487385, "epoch": 0.5602957906712173, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04329884052276611, "kl": 0.012780958786606789, "learning_rate": 3.962457337883959e-06, "loss": -0.1931, "num_tokens": 104184686.0, "reward": 0.7750000357627869, "reward_std": 0.8911962509155273, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.404819965362549, "sampling/importance_sampling_ratio/mean": 0.5695093274116516, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.65706467628479, "sampling/sampling_logp_difference/mean": 0.013516822829842567, "step": 985, "step_time": 74.13944021333009 }, { "clip_ratio/high_max": 0.012850692379288375, "clip_ratio/high_mean": 0.004389819077914581, "clip_ratio/low_mean": 0.0018644633091753349, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006254282227018848, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 819.5250244140625, "completions/mean_terminated_length": 634.5238037109375, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.11667420901358128, "epoch": 0.5608646188850968, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01995733566582203, "kl": 0.012476324918679893, "learning_rate": 3.957337883959044e-06, "loss": -0.0461, "num_tokens": 104291235.0, "reward": 0.925000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.9655331373214722, "sampling/importance_sampling_ratio/mean": 0.2923029065132141, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.074507474899292, "sampling/sampling_logp_difference/mean": 0.014316859655082226, "step": 986, "step_time": 73.73117116210051 }, { "clip_ratio/high_max": 0.010874252824578434, "clip_ratio/high_mean": 0.002996070441440679, "clip_ratio/low_mean": 0.0045071529457345605, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.0075032233144156635, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 895.5250244140625, "completions/mean_terminated_length": 721.7058715820312, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.12540104798972607, "epoch": 0.5614334470989761, "frac_reward_zero_std": 0.0, "grad_norm": 0.02237696945667267, "kl": 0.014753807918168604, "learning_rate": 3.95221843003413e-06, "loss": 0.0604, "num_tokens": 104395128.0, "reward": 0.6500000357627869, "reward_std": 0.8335896730422974, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.829908609390259, "sampling/importance_sampling_ratio/mean": 0.4244084358215332, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8489476442337036, "sampling/sampling_logp_difference/mean": 0.015887970104813576, "step": 987, "step_time": 75.61494261166081 }, { "clip_ratio/high_max": 0.015076282201334834, "clip_ratio/high_mean": 0.005750944139435887, "clip_ratio/low_mean": 0.002064546017209068, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.007815489894710481, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 962.0, "completions/mean_length": 848.7250366210938, "completions/mean_terminated_length": 655.0, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "entropy": 0.09412322985008359, "epoch": 0.5620022753128555, "frac_reward_zero_std": 0.0, "grad_norm": 0.03592405468225479, "kl": 0.0129972894792445, "learning_rate": 3.947098976109215e-06, "loss": -0.0624, "num_tokens": 104498029.0, "reward": 0.800000011920929, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.2845094203948975, "sampling/importance_sampling_ratio/mean": 0.46521949768066406, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.498704195022583, "sampling/sampling_logp_difference/mean": 0.012576768174767494, "step": 988, "step_time": 74.06565099861473 }, { "clip_ratio/high_max": 0.012025697040371597, "clip_ratio/high_mean": 0.0034307072055526078, "clip_ratio/low_mean": 0.0023681640304857865, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005798871163278818, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 971.0, "completions/mean_length": 886.2250366210938, "completions/mean_terminated_length": 564.75, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.08856676239520311, "epoch": 0.5625711035267349, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015590237453579903, "kl": 0.01691417582333088, "learning_rate": 3.9419795221843006e-06, "loss": 0.0155, "num_tokens": 104608990.0, "reward": 0.6000000238418579, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.880352020263672, "sampling/importance_sampling_ratio/mean": 0.4056673049926758, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.580200672149658, "sampling/sampling_logp_difference/mean": 0.011709398590028286, "step": 989, "step_time": 74.68704567081295 }, { "clip_ratio/high_max": 0.006696387834381312, "clip_ratio/high_mean": 0.0015584807988489047, "clip_ratio/low_mean": 0.0018554687630967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003413949569221586, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 949.0, "completions/mean_length": 758.9500122070312, "completions/mean_terminated_length": 542.0909423828125, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.10892399679869413, "epoch": 0.5631399317406144, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02251933328807354, "kl": 0.0147856161929667, "learning_rate": 3.936860068259385e-06, "loss": -0.0424, "num_tokens": 104717612.0, "reward": 0.925000011920929, "reward_std": 0.9167249798774719, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.8021893501281738, "sampling/importance_sampling_ratio/mean": 0.4288858473300934, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7742578983306885, "sampling/sampling_logp_difference/mean": 0.01475592888891697, "step": 990, "step_time": 73.8449864112772 }, { "clip_ratio/high_max": 0.00998403102857992, "clip_ratio/high_mean": 0.002446051948936656, "clip_ratio/low_mean": 0.0016812701032904442, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00412732208496891, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 854.0, "completions/mean_length": 934.7750244140625, "completions/mean_terminated_length": 667.1000366210938, "completions/min_length": 403.0, "completions/min_terminated_length": 403.0, "entropy": 0.10223833564668894, "epoch": 0.5637087599544938, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.009661502204835415, "kl": 0.012973191449418664, "learning_rate": 3.9317406143344714e-06, "loss": 0.0053, "num_tokens": 104826931.0, "reward": 0.4749999940395355, "reward_std": 0.8469401597976685, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.5936009883880615, "sampling/importance_sampling_ratio/mean": 0.39953532814979553, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.600684642791748, "sampling/sampling_logp_difference/mean": 0.012995988130569458, "step": 991, "step_time": 73.33990181726404 }, { "clip_ratio/high_max": 0.009657379589043558, "clip_ratio/high_mean": 0.002393994669546373, "clip_ratio/low_mean": 0.001185088010970503, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035790826368611306, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 892.125, "completions/mean_terminated_length": 618.2307739257812, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.07583161676302552, "epoch": 0.5642775881683731, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013388649560511112, "kl": 0.012639786466024816, "learning_rate": 3.926621160409556e-06, "loss": -0.0138, "num_tokens": 104923496.0, "reward": 0.6000000238418579, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.2086687088012695, "sampling/importance_sampling_ratio/mean": 0.5142322778701782, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0583362579345703, "sampling/sampling_logp_difference/mean": 0.010302763432264328, "step": 992, "step_time": 72.184095174307 }, { "clip_ratio/high_max": 0.014924821793101728, "clip_ratio/high_mean": 0.005111867154482752, "clip_ratio/low_mean": 0.0021862144567421637, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007298081705812365, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 930.0, "completions/mean_length": 867.5250244140625, "completions/mean_terminated_length": 676.2777709960938, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "entropy": 0.14902853406965733, "epoch": 0.5648464163822525, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.012550149112939835, "kl": 0.014738646103069186, "learning_rate": 3.9215017064846414e-06, "loss": -0.0298, "num_tokens": 105031621.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.0809054374694824, "sampling/importance_sampling_ratio/mean": 0.3160683512687683, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7560703754425049, "sampling/sampling_logp_difference/mean": 0.016453241929411888, "step": 993, "step_time": 74.70081605180167 }, { "clip_ratio/high_max": 0.008493632398312911, "clip_ratio/high_mean": 0.002454231525916839, "clip_ratio/low_mean": 0.0012695312580035534, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003723762827576138, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 860.0, "completions/mean_length": 666.375, "completions/mean_terminated_length": 494.1851806640625, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.10241758916527033, "epoch": 0.565415244596132, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0704011619091034, "kl": 0.015577320009469986, "learning_rate": 3.916382252559727e-06, "loss": -0.1277, "num_tokens": 105137548.0, "reward": 1.274999976158142, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.9111194610595703, "sampling/importance_sampling_ratio/mean": 0.5074175596237183, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6064910888671875, "sampling/sampling_logp_difference/mean": 0.014807638712227345, "step": 994, "step_time": 72.23145091068 }, { "clip_ratio/high_max": 0.010716439835960045, "clip_ratio/high_mean": 0.0030719737405888736, "clip_ratio/low_mean": 0.002343750005820766, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.00541572377551347, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 916.4750366210938, "completions/mean_terminated_length": 665.5833740234375, "completions/min_length": 421.0, "completions/min_terminated_length": 421.0, "entropy": 0.0829066950827837, "epoch": 0.5659840728100114, "frac_reward_zero_std": 0.0, "grad_norm": 0.038442447781562805, "kl": 0.015396933304145932, "learning_rate": 3.911262798634812e-06, "loss": 0.061, "num_tokens": 105249535.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.6112513542175293, "sampling/importance_sampling_ratio/mean": 0.5693908929824829, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.81429386138916, "sampling/sampling_logp_difference/mean": 0.01063742209225893, "step": 995, "step_time": 76.2632962539792 }, { "clip_ratio/high_max": 0.015959129319526255, "clip_ratio/high_mean": 0.005770904186647385, "clip_ratio/low_mean": 0.0017823688831413165, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007553272938821465, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 672.4750366210938, "completions/mean_terminated_length": 570.4193115234375, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.1202692799270153, "epoch": 0.5665529010238908, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.031326472759246826, "kl": 0.01722087466623634, "learning_rate": 3.906143344709898e-06, "loss": -0.0299, "num_tokens": 105355978.0, "reward": 1.4500000476837158, "reward_std": 0.8458041548728943, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.517449378967285, "sampling/importance_sampling_ratio/mean": 0.5205460786819458, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.604325532913208, "sampling/sampling_logp_difference/mean": 0.016833854839205742, "step": 996, "step_time": 72.74671562737785 }, { "clip_ratio/high_max": 0.01400419615674764, "clip_ratio/high_mean": 0.0034912360715679824, "clip_ratio/low_mean": 0.0022705078044964466, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00576174387242645, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 858.0, "completions/mean_length": 942.7999877929688, "completions/mean_terminated_length": 699.2000122070312, "completions/min_length": 580.0, "completions/min_terminated_length": 580.0, "entropy": 0.11909908521920443, "epoch": 0.5671217292377702, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026910699903964996, "kl": 0.012625711155124009, "learning_rate": 3.901023890784983e-06, "loss": -0.0327, "num_tokens": 105471154.0, "reward": 0.5, "reward_std": 0.8770580291748047, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.1787335872650146, "sampling/importance_sampling_ratio/mean": 0.28709957003593445, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5197829008102417, "sampling/sampling_logp_difference/mean": 0.014339643530547619, "step": 997, "step_time": 76.11287750327028 }, { "clip_ratio/high_max": 0.010034985141828656, "clip_ratio/high_mean": 0.0036028501635883003, "clip_ratio/low_mean": 0.001485720815253444, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0050885710516013205, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 898.1500244140625, "completions/mean_terminated_length": 727.8823852539062, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.07424299791455269, "epoch": 0.5676905574516496, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.036525316536426544, "kl": 0.02302245981991291, "learning_rate": 3.895904436860069e-06, "loss": 0.1195, "num_tokens": 105574656.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.6169321537017822, "sampling/importance_sampling_ratio/mean": 0.5007286071777344, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6763439178466797, "sampling/sampling_logp_difference/mean": 0.010467462241649628, "step": 998, "step_time": 73.0631023843307 }, { "clip_ratio/high_max": 0.014714884688146412, "clip_ratio/high_mean": 0.004888262519671116, "clip_ratio/low_mean": 0.002382521313847974, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007270783738931641, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 944.0, "completions/mean_length": 739.7250366210938, "completions/mean_terminated_length": 507.1363830566406, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.08990296069532633, "epoch": 0.568259385665529, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.051911961287260056, "kl": 0.016265907674096525, "learning_rate": 3.890784982935154e-06, "loss": -0.108, "num_tokens": 105681061.0, "reward": 0.9000000357627869, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.76715350151062, "sampling/importance_sampling_ratio/mean": 0.599422812461853, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2226970195770264, "sampling/sampling_logp_difference/mean": 0.012697902508080006, "step": 999, "step_time": 72.67479569255374 }, { "clip_ratio/high_max": 0.011715831002220511, "clip_ratio/high_mean": 0.0029065292328596115, "clip_ratio/low_mean": 0.0038574219215661287, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.006763951270841062, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 945.6749877929688, "completions/mean_terminated_length": 739.1818237304688, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.11560105718672276, "epoch": 0.5688282138794084, "frac_reward_zero_std": 0.0, "grad_norm": 0.04151434823870659, "kl": 0.013368355110287666, "learning_rate": 3.885665529010239e-06, "loss": 0.0571, "num_tokens": 105794296.0, "reward": 0.45000001788139343, "reward_std": 0.7828285098075867, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.7107672691345215, "sampling/importance_sampling_ratio/mean": 0.4124917984008789, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4499154090881348, "sampling/sampling_logp_difference/mean": 0.014193124137818813, "step": 1000, "step_time": 75.46346029499546 }, { "clip_ratio/high_max": 0.014127933303825557, "clip_ratio/high_mean": 0.005404288065619767, "clip_ratio/low_mean": 0.0024169922253349796, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007821280101779848, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 870.0, "completions/mean_length": 698.6500244140625, "completions/mean_terminated_length": 404.2857360839844, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.11593652237206697, "epoch": 0.5693970420932878, "frac_reward_zero_std": 0.0, "grad_norm": 0.028087327256798744, "kl": 0.022739503881894052, "learning_rate": 3.880546075085324e-06, "loss": 0.0598, "num_tokens": 105890378.0, "reward": 1.0, "reward_std": 0.9870963096618652, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.643395185470581, "sampling/importance_sampling_ratio/mean": 0.5252860188484192, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.675062894821167, "sampling/sampling_logp_difference/mean": 0.013952766545116901, "step": 1001, "step_time": 71.33918818808161 }, { "clip_ratio/high_max": 0.013338389398995787, "clip_ratio/high_mean": 0.0036338071076897904, "clip_ratio/low_mean": 0.0015860058920225129, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005219812941504642, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 943.0, "completions/mean_length": 854.0750122070312, "completions/mean_terminated_length": 501.15386962890625, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.12871740944683552, "epoch": 0.5699658703071673, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014727973379194736, "kl": 0.015829607960768044, "learning_rate": 3.8754266211604095e-06, "loss": 0.0111, "num_tokens": 106004477.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.0842621326446533, "sampling/importance_sampling_ratio/mean": 0.4598826467990875, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4179141521453857, "sampling/sampling_logp_difference/mean": 0.015135868452489376, "step": 1002, "step_time": 75.91463978611864 }, { "clip_ratio/high_max": 0.012752836337313056, "clip_ratio/high_mean": 0.0052043559844605625, "clip_ratio/low_mean": 0.0018554687412688509, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0070598247984889895, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 885.3250122070312, "completions/mean_terminated_length": 654.2000122070312, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.14672277588397264, "epoch": 0.5705346985210467, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02821122668683529, "kl": 0.017478353693149984, "learning_rate": 3.870307167235495e-06, "loss": -0.038, "num_tokens": 106112026.0, "reward": 0.625, "reward_std": 0.8678739070892334, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.8824620246887207, "sampling/importance_sampling_ratio/mean": 0.37431272864341736, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7991461753845215, "sampling/sampling_logp_difference/mean": 0.016262728720903397, "step": 1003, "step_time": 75.10233831685036 }, { "clip_ratio/high_max": 0.01766161946579814, "clip_ratio/high_mean": 0.007522877567680553, "clip_ratio/low_mean": 0.002244329734821804, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009767207433469594, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 721.9750366210938, "completions/mean_terminated_length": 540.760009765625, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.11941547691822052, "epoch": 0.571103526734926, "frac_reward_zero_std": 0.0, "grad_norm": 0.06892382353544235, "kl": 0.01710108551196754, "learning_rate": 3.86518771331058e-06, "loss": 0.2329, "num_tokens": 106205865.0, "reward": 1.149999976158142, "reward_std": 0.9486833810806274, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.3402886390686035, "sampling/importance_sampling_ratio/mean": 0.4288090169429779, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.021493911743164, "sampling/sampling_logp_difference/mean": 0.01618756353855133, "step": 1004, "step_time": 73.77538912533782 }, { "clip_ratio/high_max": 0.013240591099020094, "clip_ratio/high_mean": 0.005209336115512997, "clip_ratio/low_mean": 0.0025801895681070164, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007789525494445115, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 815.75, "completions/mean_terminated_length": 676.9166870117188, "completions/min_length": 441.0, "completions/min_terminated_length": 441.0, "entropy": 0.1231697229668498, "epoch": 0.5716723549488054, "frac_reward_zero_std": 0.0, "grad_norm": 0.03829950839281082, "kl": 0.015093057532794774, "learning_rate": 3.860068259385666e-06, "loss": -0.1503, "num_tokens": 106304767.0, "reward": 1.0, "reward_std": 0.9058216214179993, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.855360269546509, "sampling/importance_sampling_ratio/mean": 0.47107356786727905, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3139431476593018, "sampling/sampling_logp_difference/mean": 0.015623371116816998, "step": 1005, "step_time": 71.46441491390578 }, { "clip_ratio/high_max": 0.015705730358604342, "clip_ratio/high_mean": 0.005322309385519475, "clip_ratio/low_mean": 0.0007609666863572784, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0060832760646007955, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 771.4750366210938, "completions/mean_terminated_length": 726.9117431640625, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "entropy": 0.14794256445020437, "epoch": 0.5722411831626849, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.012639418244361877, "kl": 0.017958702868781984, "learning_rate": 3.854948805460751e-06, "loss": -0.0082, "num_tokens": 106409458.0, "reward": 1.524999976158142, "reward_std": 0.7506407499313354, "rewards/format_reward/mean": 0.8500000238418579, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.8689355850219727, "sampling/importance_sampling_ratio/mean": 0.42375677824020386, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9736307859420776, "sampling/sampling_logp_difference/mean": 0.017411893233656883, "step": 1006, "step_time": 71.03776601073332 }, { "clip_ratio/high_max": 0.011543736676685512, "clip_ratio/high_mean": 0.0036117835843469948, "clip_ratio/low_mean": 0.0019015206707990728, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005513304175110534, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 950.0, "completions/mean_length": 740.0750122070312, "completions/mean_terminated_length": 669.09375, "completions/min_length": 489.0, "completions/min_terminated_length": 489.0, "entropy": 0.1433978984132409, "epoch": 0.5728100113765643, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011492233723402023, "kl": 0.01643211836926639, "learning_rate": 3.849829351535837e-06, "loss": -0.0208, "num_tokens": 106509413.0, "reward": 1.4500000476837158, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.9149210453033447, "sampling/importance_sampling_ratio/mean": 0.23025712370872498, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1808090209960938, "sampling/sampling_logp_difference/mean": 0.01797364093363285, "step": 1007, "step_time": 71.57644234411418 }, { "clip_ratio/high_max": 0.012414881202857941, "clip_ratio/high_mean": 0.004032850018120371, "clip_ratio/low_mean": 0.0030761718517169356, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.0071090217679739, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 772.0, "completions/mean_length": 865.2999877929688, "completions/mean_terminated_length": 570.5714721679688, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.10566593892872334, "epoch": 0.5733788395904437, "frac_reward_zero_std": 0.0, "grad_norm": 0.04164106026291847, "kl": 0.01361125207040459, "learning_rate": 3.844709897610921e-06, "loss": -0.0935, "num_tokens": 106615473.0, "reward": 0.574999988079071, "reward_std": 0.8439072966575623, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.7806308269500732, "sampling/importance_sampling_ratio/mean": 0.4958718419075012, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9589037895202637, "sampling/sampling_logp_difference/mean": 0.01284611877053976, "step": 1008, "step_time": 75.52720465534367 }, { "clip_ratio/high_max": 0.015422371914610267, "clip_ratio/high_mean": 0.0045397783105727285, "clip_ratio/low_mean": 0.0017089843549911166, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006248762831091881, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 905.0250244140625, "completions/mean_terminated_length": 706.7333984375, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.0856982646510005, "epoch": 0.573947667804323, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02497897297143936, "kl": 0.013785384478978813, "learning_rate": 3.839590443686007e-06, "loss": -0.0186, "num_tokens": 106725378.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.9872238636016846, "sampling/importance_sampling_ratio/mean": 0.44478440284729004, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4688422679901123, "sampling/sampling_logp_difference/mean": 0.011203872971236706, "step": 1009, "step_time": 74.49738888256252 }, { "clip_ratio/high_max": 0.006296783802099526, "clip_ratio/high_mean": 0.0012593567662406713, "clip_ratio/low_mean": 0.0006835937510913936, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001942950504599139, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 803.0, "completions/mean_length": 757.4500122070312, "completions/mean_terminated_length": 490.8999938964844, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.10217108018696308, "epoch": 0.5745164960182025, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01134667918086052, "kl": 0.014764562249183655, "learning_rate": 3.834470989761092e-06, "loss": 0.0337, "num_tokens": 106828132.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.8166351318359375, "sampling/importance_sampling_ratio/mean": 0.4402996003627777, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.796236276626587, "sampling/sampling_logp_difference/mean": 0.014567677862942219, "step": 1010, "step_time": 72.1515059878584 }, { "clip_ratio/high_max": 0.011433165054768324, "clip_ratio/high_mean": 0.0031841179297771305, "clip_ratio/low_mean": 0.0016694942096364684, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00485361207392998, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 656.0, "completions/mean_length": 856.2250366210938, "completions/mean_terminated_length": 507.7692565917969, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.11806839797645807, "epoch": 0.5750853242320819, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.034158725291490555, "kl": 0.012645494425669312, "learning_rate": 3.8293515358361776e-06, "loss": -0.0452, "num_tokens": 106935133.0, "reward": 0.574999988079071, "reward_std": 0.8737629055976868, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.750598192214966, "sampling/importance_sampling_ratio/mean": 0.47873130440711975, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9746894836425781, "sampling/sampling_logp_difference/mean": 0.013639452867209911, "step": 1011, "step_time": 71.89442120702006 }, { "clip_ratio/high_max": 0.012908144912216812, "clip_ratio/high_mean": 0.003944985364796594, "clip_ratio/low_mean": 0.0022216797078726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00616666508722119, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 920.1500244140625, "completions/mean_terminated_length": 727.2857666015625, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.10967998020350933, "epoch": 0.5756541524459613, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014887181110680103, "kl": 0.01034424576209858, "learning_rate": 3.824232081911263e-06, "loss": 0.0448, "num_tokens": 107046283.0, "reward": 0.550000011920929, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.7143619060516357, "sampling/importance_sampling_ratio/mean": 0.3775208294391632, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.931779384613037, "sampling/sampling_logp_difference/mean": 0.013468449003994465, "step": 1012, "step_time": 73.51998489908874 }, { "clip_ratio/high_max": 0.012471411318983883, "clip_ratio/high_mean": 0.004717348841950297, "clip_ratio/low_mean": 0.001074218753274181, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005791567615233362, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 825.5, "completions/mean_terminated_length": 627.0, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.09211043734103441, "epoch": 0.5762229806598407, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013415475375950336, "kl": 0.015525026014074683, "learning_rate": 3.8191126279863484e-06, "loss": -0.0376, "num_tokens": 107147159.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.561220645904541, "sampling/importance_sampling_ratio/mean": 0.23500505089759827, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.874356269836426, "sampling/sampling_logp_difference/mean": 0.012704971246421337, "step": 1013, "step_time": 73.31953063211404 }, { "clip_ratio/high_max": 0.013638885808177292, "clip_ratio/high_mean": 0.004035088306409307, "clip_ratio/low_mean": 0.0010253906202706276, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005060478812083602, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 881.0, "completions/mean_length": 862.5250244140625, "completions/mean_terminated_length": 485.75, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.08489783061668277, "epoch": 0.5767918088737202, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019137518480420113, "kl": 0.01286119659198448, "learning_rate": 3.8139931740614334e-06, "loss": -0.0436, "num_tokens": 107251324.0, "reward": 0.6000000238418579, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.537182569503784, "sampling/importance_sampling_ratio/mean": 0.4877067506313324, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9121567010879517, "sampling/sampling_logp_difference/mean": 0.012095816433429718, "step": 1014, "step_time": 74.39178368938155 }, { "clip_ratio/high_max": 0.013453445513732731, "clip_ratio/high_mean": 0.006572037993464619, "clip_ratio/low_mean": 0.0015136718866415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008085709647275507, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 916.0, "completions/mean_length": 726.0750122070312, "completions/mean_terminated_length": 582.629638671875, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.09471657872200012, "epoch": 0.5773606370875995, "frac_reward_zero_std": 0.0, "grad_norm": 0.015325877815485, "kl": 0.017218732158653438, "learning_rate": 3.808873720136519e-06, "loss": -0.0652, "num_tokens": 107348175.0, "reward": 1.2000000476837158, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.5458872318267822, "sampling/importance_sampling_ratio/mean": 0.3885795772075653, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.341811180114746, "sampling/sampling_logp_difference/mean": 0.014347943477332592, "step": 1015, "step_time": 71.36279414058663 }, { "clip_ratio/high_max": 0.016702067106962204, "clip_ratio/high_mean": 0.004613273456925526, "clip_ratio/low_mean": 0.002037746275163954, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006651019735727459, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 820.0, "completions/mean_length": 857.125, "completions/mean_terminated_length": 510.5384826660156, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.07281575119122863, "epoch": 0.5779294653014789, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020613018423318863, "kl": 0.01496628881432116, "learning_rate": 3.8037542662116043e-06, "loss": 0.0229, "num_tokens": 107451172.0, "reward": 0.6000000238418579, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.7383575439453125, "sampling/importance_sampling_ratio/mean": 0.3063679337501526, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.958310842514038, "sampling/sampling_logp_difference/mean": 0.010715357959270477, "step": 1016, "step_time": 71.87011229991913 }, { "clip_ratio/high_max": 0.018227945547550917, "clip_ratio/high_mean": 0.00440875995263923, "clip_ratio/low_mean": 0.0011718750101863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005580634897341952, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 843.0, "completions/mean_length": 873.7750244140625, "completions/mean_terminated_length": 477.727294921875, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.08836120739579201, "epoch": 0.5784982935153583, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02470632828772068, "kl": 0.017531836521811783, "learning_rate": 3.7986348122866897e-06, "loss": -0.0264, "num_tokens": 107554779.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.693692207336426, "sampling/importance_sampling_ratio/mean": 0.38205575942993164, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7366530895233154, "sampling/sampling_logp_difference/mean": 0.011880885809659958, "step": 1017, "step_time": 71.60697393096052 }, { "clip_ratio/high_max": 0.006592840363737196, "clip_ratio/high_mean": 0.0014625773110310547, "clip_ratio/low_mean": 0.0021972656177240424, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003659842914203182, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 998.7750244140625, "completions/mean_terminated_length": 855.8333740234375, "completions/min_length": 508.0, "completions/min_terminated_length": 508.0, "entropy": 0.07450299244374037, "epoch": 0.5790671217292378, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.026682520285248756, "kl": 0.0092869694926776, "learning_rate": 3.7935153583617748e-06, "loss": -0.0057, "num_tokens": 107667058.0, "reward": 0.25, "reward_std": 0.6304251551628113, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.354956865310669, "sampling/importance_sampling_ratio/mean": 0.40651020407676697, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.25791072845459, "sampling/sampling_logp_difference/mean": 0.010179207660257816, "step": 1018, "step_time": 74.92097053304315 }, { "clip_ratio/high_max": 0.005298389412928373, "clip_ratio/high_mean": 0.0010596779029583558, "clip_ratio/low_mean": 0.0020751952979480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00313487317180261, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 909.0, "completions/mean_length": 886.1000366210938, "completions/mean_terminated_length": 522.5454711914062, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.0723097138106823, "epoch": 0.5796359499431172, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.026052335277199745, "kl": 0.01060339086689055, "learning_rate": 3.7883959044368606e-06, "loss": 0.0179, "num_tokens": 107779782.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.2577056884765625, "sampling/importance_sampling_ratio/mean": 0.46474725008010864, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9270360469818115, "sampling/sampling_logp_difference/mean": 0.01143928524106741, "step": 1019, "step_time": 73.52630855562165 }, { "clip_ratio/high_max": 0.017267670773435384, "clip_ratio/high_mean": 0.006743327510775998, "clip_ratio/low_mean": 0.0016601562383584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00840348398196511, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 832.0, "completions/mean_length": 751.9750366210938, "completions/mean_terminated_length": 505.8571472167969, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.09569337498396635, "epoch": 0.5802047781569966, "frac_reward_zero_std": 0.0, "grad_norm": 0.025989271700382233, "kl": 0.020838241442106664, "learning_rate": 3.7832764505119456e-06, "loss": -0.1143, "num_tokens": 107887661.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057363510131836, "sampling/importance_sampling_ratio/max": 2.9522545337677, "sampling/importance_sampling_ratio/mean": 0.4902285933494568, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.313490867614746, "sampling/sampling_logp_difference/mean": 0.013498643413186073, "step": 1020, "step_time": 74.68283396586776 }, { "clip_ratio/high_max": 0.015166309545747936, "clip_ratio/high_mean": 0.0044250818027649075, "clip_ratio/low_mean": 0.002356197335757315, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0067812789347954094, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 873.9750366210938, "completions/mean_terminated_length": 623.933349609375, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.1172940582036972, "epoch": 0.580773606370876, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.016069279983639717, "kl": 0.01626330951694399, "learning_rate": 3.7781569965870306e-06, "loss": 0.0093, "num_tokens": 107996156.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.653438687324524, "sampling/importance_sampling_ratio/mean": 0.25949451327323914, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.848389148712158, "sampling/sampling_logp_difference/mean": 0.014798945747315884, "step": 1021, "step_time": 73.54655564925633 }, { "clip_ratio/high_max": 0.01215559768024832, "clip_ratio/high_mean": 0.002431119588436559, "clip_ratio/low_mean": 0.001538085940410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003969205426983535, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 776.0, "completions/mean_length": 969.0499877929688, "completions/mean_terminated_length": 657.6666870117188, "completions/min_length": 585.0, "completions/min_terminated_length": 585.0, "entropy": 0.17464740946888924, "epoch": 0.5813424345847554, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.024715691804885864, "kl": 0.014455180265940726, "learning_rate": 3.773037542662116e-06, "loss": 0.0674, "num_tokens": 108107926.0, "reward": 0.17499999701976776, "reward_std": 0.4464963376522064, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.7039332389831543, "sampling/importance_sampling_ratio/mean": 0.411198228597641, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7858777046203613, "sampling/sampling_logp_difference/mean": 0.01756737194955349, "step": 1022, "step_time": 76.49936972535215 }, { "clip_ratio/high_max": 0.013943077996373177, "clip_ratio/high_mean": 0.0047110932355280966, "clip_ratio/low_mean": 0.0014188580789777916, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006129951332695782, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 692.5, "completions/mean_terminated_length": 550.4285888671875, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.11645019240677357, "epoch": 0.5819112627986348, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022093523293733597, "kl": 0.017932646325789392, "learning_rate": 3.767918088737201e-06, "loss": -0.0469, "num_tokens": 108215114.0, "reward": 1.3250000476837158, "reward_std": 0.9167249798774719, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.7253066301345825, "sampling/importance_sampling_ratio/mean": 0.26488175988197327, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1515955924987793, "sampling/sampling_logp_difference/mean": 0.015266567468643188, "step": 1023, "step_time": 72.45498949754983 }, { "clip_ratio/high_max": 0.008374041295610368, "clip_ratio/high_mean": 0.0020923822085023858, "clip_ratio/low_mean": 0.0003662109447759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024585931532783434, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 610.0, "completions/mean_length": 794.0, "completions/mean_terminated_length": 512.888916015625, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.09665344469249249, "epoch": 0.5824800910125142, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.018581915646791458, "kl": 0.016221032477915287, "learning_rate": 3.762798634812287e-06, "loss": 0.0421, "num_tokens": 108317146.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.6529552936553955, "sampling/importance_sampling_ratio/mean": 0.33218860626220703, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1184353828430176, "sampling/sampling_logp_difference/mean": 0.01377419289201498, "step": 1024, "step_time": 73.16619577817619 }, { "clip_ratio/high_max": 0.01094391368678771, "clip_ratio/high_mean": 0.003919281116395723, "clip_ratio/low_mean": 0.002594680860056542, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006513961881864816, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 761.8500366210938, "completions/mean_terminated_length": 568.0869750976562, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.097542405128479, "epoch": 0.5830489192263937, "frac_reward_zero_std": 0.0, "grad_norm": 0.017555363476276398, "kl": 0.015586949069984257, "learning_rate": 3.757679180887372e-06, "loss": -0.0133, "num_tokens": 108418588.0, "reward": 0.949999988079071, "reward_std": 0.9044051170349121, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.3745393753051758, "sampling/importance_sampling_ratio/mean": 0.2758066654205322, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8378628492355347, "sampling/sampling_logp_difference/mean": 0.013454228639602661, "step": 1025, "step_time": 69.36107718991116 }, { "clip_ratio/high_max": 0.013177732354961336, "clip_ratio/high_mean": 0.004874796431977302, "clip_ratio/low_mean": 0.001367187505820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062419838504865766, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 779.0, "completions/mean_length": 651.125, "completions/mean_terminated_length": 526.8333740234375, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.1089341901242733, "epoch": 0.5836177474402731, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04160039871931076, "kl": 0.022698294604197145, "learning_rate": 3.7525597269624574e-06, "loss": 0.0769, "num_tokens": 108528281.0, "reward": 1.3250000476837158, "reward_std": 0.8589648008346558, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.6701176166534424, "sampling/importance_sampling_ratio/mean": 0.49737754464149475, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4231677055358887, "sampling/sampling_logp_difference/mean": 0.016256654635071754, "step": 1026, "step_time": 74.98896594787948 }, { "clip_ratio/high_max": 0.014175082673318684, "clip_ratio/high_mean": 0.0038199720438569784, "clip_ratio/low_mean": 0.002978515563881956, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.006798487680498511, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 778.0, "completions/mean_length": 906.0250244140625, "completions/mean_terminated_length": 630.75, "completions/min_length": 419.0, "completions/min_terminated_length": 419.0, "entropy": 0.09138165879994631, "epoch": 0.5841865756541524, "frac_reward_zero_std": 0.0, "grad_norm": 0.022311991080641747, "kl": 0.011679055169224739, "learning_rate": 3.7474402730375424e-06, "loss": -0.0251, "num_tokens": 108638098.0, "reward": 0.5250000357627869, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 1.5571296215057373, "sampling/importance_sampling_ratio/mean": 0.27355775237083435, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.704887628555298, "sampling/sampling_logp_difference/mean": 0.012580566108226776, "step": 1027, "step_time": 73.79287016764283 }, { "clip_ratio/high_max": 0.012667850183788687, "clip_ratio/high_mean": 0.003761042418773286, "clip_ratio/low_mean": 0.0019287109171273187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005689753277692944, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 862.9000244140625, "completions/mean_terminated_length": 563.7142944335938, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.09668550733476877, "epoch": 0.5847554038680318, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029598619788885117, "kl": 0.01617732655722648, "learning_rate": 3.7423208191126282e-06, "loss": -0.018, "num_tokens": 108730686.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.542891502380371, "sampling/importance_sampling_ratio/mean": 0.40830034017562866, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.189984083175659, "sampling/sampling_logp_difference/mean": 0.012404950335621834, "step": 1028, "step_time": 69.45856844936498 }, { "clip_ratio/high_max": 0.012401204789057374, "clip_ratio/high_mean": 0.0037310148763936013, "clip_ratio/low_mean": 0.0020019531548314262, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005732967983931303, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 923.6000366210938, "completions/mean_terminated_length": 658.9091186523438, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "entropy": 0.07688219891861081, "epoch": 0.5853242320819113, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.006742704194039106, "kl": 0.011279155733063817, "learning_rate": 3.7372013651877133e-06, "loss": 0.002, "num_tokens": 108842230.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.63198184967041, "sampling/importance_sampling_ratio/mean": 0.25162187218666077, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.216453790664673, "sampling/sampling_logp_difference/mean": 0.011001941747963428, "step": 1029, "step_time": 73.62558926222846 }, { "clip_ratio/high_max": 0.010550263919867575, "clip_ratio/high_mean": 0.002829950346495025, "clip_ratio/low_mean": 0.0014161076251184568, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00424605788430199, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 921.0, "completions/mean_length": 923.6749877929688, "completions/mean_terminated_length": 715.3077392578125, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.10492784064263105, "epoch": 0.5858930602957907, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.006153729744255543, "kl": 0.012759867706336081, "learning_rate": 3.7320819112627987e-06, "loss": -0.023, "num_tokens": 108949489.0, "reward": 0.574999988079071, "reward_std": 0.873762845993042, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.591644525527954, "sampling/importance_sampling_ratio/mean": 0.4408698081970215, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1947457790374756, "sampling/sampling_logp_difference/mean": 0.013026527129113674, "step": 1030, "step_time": 72.90566339204088 }, { "clip_ratio/high_max": 0.01597151136957109, "clip_ratio/high_mean": 0.006731044501066208, "clip_ratio/low_mean": 0.0027656524507619906, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009496696875430644, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 834.4249877929688, "completions/mean_terminated_length": 662.90478515625, "completions/min_length": 358.0, "completions/min_terminated_length": 358.0, "entropy": 0.15566450357437134, "epoch": 0.5864618885096701, "frac_reward_zero_std": 0.0, "grad_norm": 0.036664195358753204, "kl": 0.020310707157477736, "learning_rate": 3.7269624573378837e-06, "loss": -0.0202, "num_tokens": 109055938.0, "reward": 0.949999988079071, "reward_std": 0.9594335556030273, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.535853862762451, "sampling/importance_sampling_ratio/mean": 0.36701375246047974, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3389153480529785, "sampling/sampling_logp_difference/mean": 0.01853514462709427, "step": 1031, "step_time": 74.11447885422967 }, { "clip_ratio/high_max": 0.00550450746959541, "clip_ratio/high_mean": 0.0011009014851879328, "clip_ratio/low_mean": 0.0010451296548126265, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021460311327246018, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 888.7750244140625, "completions/mean_terminated_length": 685.9375, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.07213296461850405, "epoch": 0.5870307167235495, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022282974794507027, "kl": 0.012089930474758148, "learning_rate": 3.7218430034129696e-06, "loss": -0.0785, "num_tokens": 109161273.0, "reward": 0.5, "reward_std": 0.71611487865448, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 1.691975474357605, "sampling/importance_sampling_ratio/mean": 0.3727869987487793, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.276573896408081, "sampling/sampling_logp_difference/mean": 0.01131574995815754, "step": 1032, "step_time": 74.99075654498301 }, { "clip_ratio/high_max": 0.007642441662028432, "clip_ratio/high_mean": 0.002072430099360645, "clip_ratio/low_mean": 0.001000976568320766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030734066385775805, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 787.4750366210938, "completions/mean_terminated_length": 593.95458984375, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.10036202054470778, "epoch": 0.5875995449374289, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01969069242477417, "kl": 0.013557217200286686, "learning_rate": 3.7167235494880546e-06, "loss": -0.0074, "num_tokens": 109269572.0, "reward": 1.0500000715255737, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.2746098041534424, "sampling/importance_sampling_ratio/mean": 0.4907100796699524, "sampling/importance_sampling_ratio/min": 0.008196557871997356, "sampling/sampling_logp_difference/max": 1.8997561931610107, "sampling/sampling_logp_difference/mean": 0.013421893119812012, "step": 1033, "step_time": 74.10380933317356 }, { "clip_ratio/high_max": 0.007852675509639084, "clip_ratio/high_mean": 0.0017833010351751, "clip_ratio/low_mean": 0.0012695312707364792, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030528323113685474, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 940.0, "completions/mean_length": 844.3500366210938, "completions/mean_terminated_length": 471.23077392578125, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.10531573556363583, "epoch": 0.5881683731513083, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.013585522770881653, "kl": 0.012516640475951135, "learning_rate": 3.71160409556314e-06, "loss": 0.078, "num_tokens": 109374250.0, "reward": 0.6500000357627869, "reward_std": 0.9486833214759827, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.328986406326294, "sampling/importance_sampling_ratio/mean": 0.55731600522995, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.656618356704712, "sampling/sampling_logp_difference/mean": 0.013904946856200695, "step": 1034, "step_time": 70.27467788220383 }, { "clip_ratio/high_max": 0.013108279090374708, "clip_ratio/high_mean": 0.004329200062784366, "clip_ratio/low_mean": 0.0016300464667438064, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0059592466277536005, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 777.0, "completions/mean_length": 766.0499877929688, "completions/mean_terminated_length": 508.1000061035156, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.08978043776005507, "epoch": 0.5887372013651877, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014004607684910297, "kl": 0.014517152332700789, "learning_rate": 3.706484641638225e-06, "loss": -0.0258, "num_tokens": 109477340.0, "reward": 0.800000011920929, "reward_std": 0.8828856945037842, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.7422468662261963, "sampling/importance_sampling_ratio/mean": 0.4089692234992981, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0720722675323486, "sampling/sampling_logp_difference/mean": 0.01248833816498518, "step": 1035, "step_time": 72.75539683434181 }, { "clip_ratio/high_max": 0.016374402097426355, "clip_ratio/high_mean": 0.006067819311283529, "clip_ratio/low_mean": 0.0020154526519036153, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008083271910436451, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.0, "completions/mean_length": 848.0499877929688, "completions/mean_terminated_length": 633.0, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.11377093847841024, "epoch": 0.5893060295790671, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.017449049279093742, "kl": 0.012929802818689495, "learning_rate": 3.701365187713311e-06, "loss": 0.0244, "num_tokens": 109581902.0, "reward": 0.8500000238418579, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.9619475603103638, "sampling/importance_sampling_ratio/mean": 0.377108097076416, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.091719150543213, "sampling/sampling_logp_difference/mean": 0.014100435189902782, "step": 1036, "step_time": 74.02729319920763 }, { "clip_ratio/high_max": 0.006390236696461216, "clip_ratio/high_mean": 0.001404629641911015, "clip_ratio/low_mean": 0.001419437539880164, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002824067138135433, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 918.0, "completions/mean_length": 783.3500366210938, "completions/mean_terminated_length": 653.7692260742188, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.08560628723353148, "epoch": 0.5898748577929466, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.023065565153956413, "kl": 0.016269049956463277, "learning_rate": 3.696245733788396e-06, "loss": -0.0928, "num_tokens": 109685676.0, "reward": 1.25, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.5565016269683838, "sampling/importance_sampling_ratio/mean": 0.2830522656440735, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0440189838409424, "sampling/sampling_logp_difference/mean": 0.01333403866738081, "step": 1037, "step_time": 73.78460091818124 }, { "clip_ratio/high_max": 0.01155440608272329, "clip_ratio/high_mean": 0.0035489370638970286, "clip_ratio/low_mean": 0.0017952402122318745, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005344177276128903, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 798.6749877929688, "completions/mean_terminated_length": 648.4583740234375, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "entropy": 0.09071239270269871, "epoch": 0.590443686006826, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.058670949190855026, "kl": 0.012437057681381702, "learning_rate": 3.6911262798634813e-06, "loss": -0.2295, "num_tokens": 109780799.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.720444917678833, "sampling/importance_sampling_ratio/mean": 0.634728729724884, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.349133253097534, "sampling/sampling_logp_difference/mean": 0.013016101904213428, "step": 1038, "step_time": 71.62379887700081 }, { "clip_ratio/high_max": 0.009575934673193842, "clip_ratio/high_mean": 0.002783373318379745, "clip_ratio/low_mean": 0.001220703143189894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004004076472483575, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 899.9000244140625, "completions/mean_terminated_length": 572.727294921875, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.0766423330642283, "epoch": 0.5910125142207053, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016903109848499298, "kl": 0.01311085990164429, "learning_rate": 3.6860068259385663e-06, "loss": 0.0858, "num_tokens": 109892995.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.8511440753936768, "sampling/importance_sampling_ratio/mean": 0.3885684013366699, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3428432941436768, "sampling/sampling_logp_difference/mean": 0.010602091439068317, "step": 1039, "step_time": 75.0750645310618 }, { "clip_ratio/high_max": 0.012677317135967314, "clip_ratio/high_mean": 0.004339509818237275, "clip_ratio/low_mean": 0.0009033203332364792, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005242830142378807, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.0, "completions/mean_length": 846.9000244140625, "completions/mean_terminated_length": 607.2941284179688, "completions/min_length": 358.0, "completions/min_terminated_length": 358.0, "entropy": 0.1092627439647913, "epoch": 0.5915813424345847, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.010360324755311012, "kl": 0.013750978861935437, "learning_rate": 3.680887372013652e-06, "loss": -0.0304, "num_tokens": 110004095.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.7226687669754028, "sampling/importance_sampling_ratio/mean": 0.3428896963596344, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.192007064819336, "sampling/sampling_logp_difference/mean": 0.013611065223813057, "step": 1040, "step_time": 76.27097009890713 }, { "clip_ratio/high_max": 0.010734597453847528, "clip_ratio/high_mean": 0.002146919519873336, "clip_ratio/low_mean": 0.001977539053768851, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004124458529986441, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 998.9750366210938, "completions/mean_terminated_length": 823.7999877929688, "completions/min_length": 585.0, "completions/min_terminated_length": 585.0, "entropy": 0.07418194133788347, "epoch": 0.5921501706484642, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013741733506321907, "kl": 0.007746410847175866, "learning_rate": 3.675767918088737e-06, "loss": 0.0383, "num_tokens": 110121166.0, "reward": 0.22500000894069672, "reward_std": 0.6196566820144653, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 1.8036460876464844, "sampling/importance_sampling_ratio/mean": 0.5239735841751099, "sampling/importance_sampling_ratio/min": 0.004410912748426199, "sampling/sampling_logp_difference/max": 2.594892978668213, "sampling/sampling_logp_difference/mean": 0.009563704021275043, "step": 1041, "step_time": 77.92084810114466 }, { "clip_ratio/high_max": 0.0124214093957562, "clip_ratio/high_mean": 0.003926790624973364, "clip_ratio/low_mean": 0.0014925673403922701, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0054193579708226025, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 925.2250366210938, "completions/mean_terminated_length": 741.7857666015625, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "entropy": 0.10680830013006926, "epoch": 0.5927189988623436, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01166817918419838, "kl": 0.013897116761654615, "learning_rate": 3.6706484641638226e-06, "loss": -0.0105, "num_tokens": 110233375.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.390723943710327, "sampling/importance_sampling_ratio/mean": 0.3013432025909424, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1079249382019043, "sampling/sampling_logp_difference/mean": 0.012703174725174904, "step": 1042, "step_time": 77.93800777290016 }, { "clip_ratio/high_max": 0.01200900156982243, "clip_ratio/high_mean": 0.0033572980682947673, "clip_ratio/low_mean": 0.0009033203241415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004260618414264172, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 743.625, "completions/mean_terminated_length": 662.2257690429688, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.07012810092419386, "epoch": 0.593287827076223, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04569440707564354, "kl": 0.012483967235311866, "learning_rate": 3.665529010238908e-06, "loss": 0.1107, "num_tokens": 110332168.0, "reward": 1.5500000715255737, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.7749999761581421, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.884767770767212, "sampling/importance_sampling_ratio/mean": 0.37524276971817017, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6596877574920654, "sampling/sampling_logp_difference/mean": 0.01161979790776968, "step": 1043, "step_time": 73.68307556491345 }, { "clip_ratio/high_max": 0.002108262327965349, "clip_ratio/high_mean": 0.000482258532429114, "clip_ratio/low_mean": 0.0019531249854480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024353835178771988, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 803.7250366210938, "completions/mean_terminated_length": 560.26318359375, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.12900394294410944, "epoch": 0.5938566552901023, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.010524398647248745, "kl": 0.01515082479454577, "learning_rate": 3.6604095563139935e-06, "loss": 0.0265, "num_tokens": 110435453.0, "reward": 0.925000011920929, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.8663675785064697, "sampling/importance_sampling_ratio/mean": 0.3588668704032898, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.232998847961426, "sampling/sampling_logp_difference/mean": 0.015585740096867085, "step": 1044, "step_time": 70.21210816316307 }, { "clip_ratio/high_max": 0.011483191628940403, "clip_ratio/high_mean": 0.0031940104818204418, "clip_ratio/low_mean": 0.0009521484353172127, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0041461588407400995, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 975.0, "completions/mean_length": 909.4000244140625, "completions/mean_terminated_length": 696.5714721679688, "completions/min_length": 552.0, "completions/min_terminated_length": 552.0, "entropy": 0.07205225713551044, "epoch": 0.5944254835039818, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.012433350086212158, "kl": 0.011448514414951205, "learning_rate": 3.6552901023890785e-06, "loss": -0.0386, "num_tokens": 110545765.0, "reward": 0.5250000357627869, "reward_std": 0.7840558886528015, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.5551717281341553, "sampling/importance_sampling_ratio/mean": 0.4304521083831787, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.636373519897461, "sampling/sampling_logp_difference/mean": 0.010089069604873657, "step": 1045, "step_time": 75.04165833769366 }, { "clip_ratio/high_max": 0.007991520571522415, "clip_ratio/high_mean": 0.0025609597942093387, "clip_ratio/low_mean": 0.001647683009650791, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004208642785670236, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 812.4249877929688, "completions/mean_terminated_length": 656.0435180664062, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.06960889371111989, "epoch": 0.5949943117178612, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.034616466611623764, "kl": 0.010664689354598522, "learning_rate": 3.650170648464164e-06, "loss": 0.0157, "num_tokens": 110634534.0, "reward": 0.824999988079071, "reward_std": 0.8129560351371765, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.282254934310913, "sampling/importance_sampling_ratio/mean": 0.4324191212654114, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7859411239624023, "sampling/sampling_logp_difference/mean": 0.010815821588039398, "step": 1046, "step_time": 124.18116266513243 }, { "clip_ratio/high_max": 0.011182083690073341, "clip_ratio/high_mean": 0.0030208318785298616, "clip_ratio/low_mean": 0.0004882812572759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035091131139779463, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 845.8500366210938, "completions/mean_terminated_length": 648.9473876953125, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.06738225882872939, "epoch": 0.5955631399317406, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.016465255990624428, "kl": 0.011142420291434973, "learning_rate": 3.6450511945392494e-06, "loss": -0.0493, "num_tokens": 110734240.0, "reward": 0.925000011920929, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.89530873298645, "sampling/importance_sampling_ratio/mean": 0.5161017179489136, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4718210697174072, "sampling/sampling_logp_difference/mean": 0.01022365689277649, "step": 1047, "step_time": 70.66896327445284 }, { "clip_ratio/high_max": 0.0031358037958852947, "clip_ratio/high_mean": 0.0007752650126349181, "clip_ratio/low_mean": 0.0011962890639551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019715540984179825, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 844.0, "completions/mean_length": 771.6000366210938, "completions/mean_terminated_length": 519.2000122070312, "completions/min_length": 393.0, "completions/min_terminated_length": 393.0, "entropy": 0.07091288967058063, "epoch": 0.59613196814562, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.0114836934953928, "kl": 0.013791065895929933, "learning_rate": 3.639931740614335e-06, "loss": 0.0415, "num_tokens": 110839792.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.359393358230591, "sampling/importance_sampling_ratio/mean": 0.46355897188186646, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.222665786743164, "sampling/sampling_logp_difference/mean": 0.011399777606129646, "step": 1048, "step_time": 72.30842910567299 }, { "clip_ratio/high_max": 0.002663883613422513, "clip_ratio/high_mean": 0.0005327767285052687, "clip_ratio/low_mean": 0.0009765625072759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015093392357812263, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 783.0, "completions/mean_length": 912.4750366210938, "completions/mean_terminated_length": 577.9000244140625, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.14279966708272696, "epoch": 0.5967007963594995, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.01876031793653965, "kl": 0.014191204449161887, "learning_rate": 3.63481228668942e-06, "loss": 0.0251, "num_tokens": 110950979.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.3917529582977295, "sampling/importance_sampling_ratio/mean": 0.3444989323616028, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7508466243743896, "sampling/sampling_logp_difference/mean": 0.01640935055911541, "step": 1049, "step_time": 74.11036057816818 }, { "clip_ratio/high_max": 0.0140818226209376, "clip_ratio/high_mean": 0.0037429694639286026, "clip_ratio/low_mean": 0.0024902343611756805, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006233203923329711, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 952.0, "completions/mean_length": 951.6500244140625, "completions/mean_terminated_length": 734.6000366210938, "completions/min_length": 527.0, "completions/min_terminated_length": 527.0, "entropy": 0.09662669245153666, "epoch": 0.5972696245733788, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02264244481921196, "kl": 0.016260008327662945, "learning_rate": 3.6296928327645053e-06, "loss": 0.0308, "num_tokens": 111067693.0, "reward": 0.45000001788139343, "reward_std": 0.7828285098075867, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.297982931137085, "sampling/importance_sampling_ratio/mean": 0.361794650554657, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3378918170928955, "sampling/sampling_logp_difference/mean": 0.012750375084578991, "step": 1050, "step_time": 77.07955196988769 }, { "clip_ratio/high_max": 0.013278493075631559, "clip_ratio/high_mean": 0.0038331542746163905, "clip_ratio/low_mean": 0.003318382689030841, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00715153687633574, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 962.0, "completions/mean_length": 858.2999877929688, "completions/mean_terminated_length": 634.11767578125, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.1383837815374136, "epoch": 0.5978384527872582, "frac_reward_zero_std": 0.0, "grad_norm": 0.04961513727903366, "kl": 0.014415579265914857, "learning_rate": 3.6245733788395907e-06, "loss": -0.0441, "num_tokens": 111171665.0, "reward": 0.7750000357627869, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.84342098236084, "sampling/importance_sampling_ratio/mean": 0.44646158814430237, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8796641826629639, "sampling/sampling_logp_difference/mean": 0.016212332993745804, "step": 1051, "step_time": 71.59822941897437 }, { "clip_ratio/high_max": 0.01470323815010488, "clip_ratio/high_mean": 0.005804283035104163, "clip_ratio/low_mean": 0.0019008073722943664, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007705090334638953, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 781.0, "completions/mean_length": 657.4750366210938, "completions/mean_terminated_length": 500.39288330078125, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.06791935674846172, "epoch": 0.5984072810011376, "frac_reward_zero_std": 0.0, "grad_norm": 0.05119357258081436, "kl": 0.013558255624957383, "learning_rate": 3.619453924914676e-06, "loss": 0.0384, "num_tokens": 111279276.0, "reward": 1.1750000715255737, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.2621994018554688, "sampling/importance_sampling_ratio/mean": 0.4369794726371765, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1272919178009033, "sampling/sampling_logp_difference/mean": 0.011741774156689644, "step": 1052, "step_time": 73.98474948480725 }, { "clip_ratio/high_max": 0.01425412145908922, "clip_ratio/high_mean": 0.005617947317659855, "clip_ratio/low_mean": 0.001979054441108019, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007597001676913351, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 728.6000366210938, "completions/mean_terminated_length": 486.90911865234375, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.08817016147077084, "epoch": 0.5989761092150171, "frac_reward_zero_std": 0.0, "grad_norm": 0.04520370066165924, "kl": 0.014517579576931894, "learning_rate": 3.614334470989761e-06, "loss": 0.1874, "num_tokens": 111382764.0, "reward": 1.0750000476837158, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.447563409805298, "sampling/importance_sampling_ratio/mean": 0.49682527780532837, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9496855735778809, "sampling/sampling_logp_difference/mean": 0.012900236062705517, "step": 1053, "step_time": 71.50305905356072 }, { "clip_ratio/high_max": 0.0037245296698529273, "clip_ratio/high_mean": 0.0007449059339705855, "clip_ratio/low_mean": 0.001806640619179234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0025515465531498194, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.0, "completions/mean_length": 765.1000366210938, "completions/mean_terminated_length": 506.20001220703125, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.09291468188166618, "epoch": 0.5995449374288965, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.008298325352370739, "kl": 0.012373823672533035, "learning_rate": 3.6092150170648466e-06, "loss": 0.086, "num_tokens": 111489248.0, "reward": 0.9750000238418579, "reward_std": 0.9996795058250427, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.3336522579193115, "sampling/importance_sampling_ratio/mean": 0.36587539315223694, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.934126377105713, "sampling/sampling_logp_difference/mean": 0.013075524941086769, "step": 1054, "step_time": 131.48303785710596 }, { "clip_ratio/high_max": 0.008167782798409462, "clip_ratio/high_mean": 0.0030113152824924327, "clip_ratio/low_mean": 0.002339790946280118, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005351106286980212, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 913.6000366210938, "completions/mean_terminated_length": 748.0, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "entropy": 0.0869711097329855, "epoch": 0.6001137656427759, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022159986197948456, "kl": 0.009884177299682051, "learning_rate": 3.604095563139932e-06, "loss": 0.1357, "num_tokens": 111585816.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.4942020177841187, "sampling/importance_sampling_ratio/mean": 0.3476892113685608, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7590374946594238, "sampling/sampling_logp_difference/mean": 0.01078363973647356, "step": 1055, "step_time": 70.72287136409432 }, { "clip_ratio/high_max": 0.008734485134482384, "clip_ratio/high_mean": 0.002758533271844499, "clip_ratio/low_mean": 0.002869241994631011, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.005627775157336146, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 907.4000244140625, "completions/mean_terminated_length": 690.857177734375, "completions/min_length": 493.0, "completions/min_terminated_length": 493.0, "entropy": 0.0897158794105053, "epoch": 0.6006825938566553, "frac_reward_zero_std": 0.0, "grad_norm": 0.03151627629995346, "kl": 0.012329901393968612, "learning_rate": 3.598976109215017e-06, "loss": -0.0045, "num_tokens": 111690624.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.77321720123291, "sampling/importance_sampling_ratio/mean": 0.3536575436592102, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0945115089416504, "sampling/sampling_logp_difference/mean": 0.01129984762519598, "step": 1056, "step_time": 72.88332040421665 }, { "clip_ratio/high_max": 0.011205478105694056, "clip_ratio/high_mean": 0.0035022935626329854, "clip_ratio/low_mean": 0.0027879812350875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006290274846833199, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 858.375, "completions/mean_terminated_length": 708.5238037109375, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.13163070846349, "epoch": 0.6012514220705347, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03929689899086952, "kl": 0.01252183283213526, "learning_rate": 3.5938566552901024e-06, "loss": -0.0637, "num_tokens": 111790399.0, "reward": 0.824999988079071, "reward_std": 0.873762845993042, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.5394513607025146, "sampling/importance_sampling_ratio/mean": 0.3302749693393707, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.880829334259033, "sampling/sampling_logp_difference/mean": 0.015586844645440578, "step": 1057, "step_time": 72.57736560702324 }, { "clip_ratio/high_max": 0.010778033174574375, "clip_ratio/high_mean": 0.0038901110674487427, "clip_ratio/low_mean": 0.0033304047537967563, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007220515515655279, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 911.0, "completions/mean_length": 844.125, "completions/mean_terminated_length": 696.95458984375, "completions/min_length": 451.0, "completions/min_terminated_length": 451.0, "entropy": 0.14804617781192064, "epoch": 0.6018202502844141, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04752922058105469, "kl": 0.017184994532726705, "learning_rate": 3.5887372013651875e-06, "loss": -0.0902, "num_tokens": 111893316.0, "reward": 0.9000000357627869, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.8662776947021484, "sampling/importance_sampling_ratio/mean": 0.49219703674316406, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0745840072631836, "sampling/sampling_logp_difference/mean": 0.016885817050933838, "step": 1058, "step_time": 71.42869945219718 }, { "clip_ratio/high_max": 0.009980363247450441, "clip_ratio/high_mean": 0.0025721166603034362, "clip_ratio/low_mean": 0.0022312995861284435, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004803416144568473, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 979.0, "completions/mean_length": 637.1500244140625, "completions/mean_terminated_length": 524.8386840820312, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.10923959873616695, "epoch": 0.6023890784982935, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.009026223793625832, "kl": 0.014132193056866527, "learning_rate": 3.5836177474402733e-06, "loss": 0.0175, "num_tokens": 111997850.0, "reward": 1.4250000715255737, "reward_std": 0.8439072966575623, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.4048457145690918, "sampling/importance_sampling_ratio/mean": 0.2826663851737976, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.249573707580566, "sampling/sampling_logp_difference/mean": 0.015437680296599865, "step": 1059, "step_time": 73.95735421963036 }, { "clip_ratio/high_max": 0.013518955442123115, "clip_ratio/high_mean": 0.003921893861843273, "clip_ratio/low_mean": 0.0022705078044964466, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006192401750013232, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 911.0, "completions/mean_length": 882.9249877929688, "completions/mean_terminated_length": 553.75, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.11465638782829046, "epoch": 0.602957906712173, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01111550722271204, "kl": 0.01815120840910822, "learning_rate": 3.5784982935153583e-06, "loss": 0.03, "num_tokens": 112107879.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.6063497066497803, "sampling/importance_sampling_ratio/mean": 0.3045661747455597, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8103235960006714, "sampling/sampling_logp_difference/mean": 0.01380117516964674, "step": 1060, "step_time": 73.71880815457553 }, { "clip_ratio/high_max": 0.014227069565095007, "clip_ratio/high_mean": 0.005535634845728055, "clip_ratio/low_mean": 0.002222182316472754, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007757817045785487, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 845.0, "completions/mean_length": 718.6000366210938, "completions/mean_terminated_length": 492.86956787109375, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.09613898862153292, "epoch": 0.6035267349260524, "frac_reward_zero_std": 0.0, "grad_norm": 0.02826007828116417, "kl": 0.022005556849762797, "learning_rate": 3.5733788395904438e-06, "loss": -0.021, "num_tokens": 112205671.0, "reward": 1.024999976158142, "reward_std": 0.9469925761222839, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.5951383113861084, "sampling/importance_sampling_ratio/mean": 0.4090377688407898, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.462101459503174, "sampling/sampling_logp_difference/mean": 0.013394812121987343, "step": 1061, "step_time": 74.03617736767046 }, { "clip_ratio/high_max": 0.014847580285277218, "clip_ratio/high_mean": 0.004492421139730141, "clip_ratio/low_mean": 0.0017708932209643535, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006263314397074282, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 772.0, "completions/mean_length": 781.6500244140625, "completions/mean_terminated_length": 539.2999877929688, "completions/min_length": 358.0, "completions/min_terminated_length": 358.0, "entropy": 0.07143590925261378, "epoch": 0.6040955631399317, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03189340978860855, "kl": 0.01106086780782789, "learning_rate": 3.5682593856655288e-06, "loss": -0.0248, "num_tokens": 112298545.0, "reward": 0.875, "reward_std": 0.9388344883918762, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.896980047225952, "sampling/importance_sampling_ratio/mean": 0.7289086580276489, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1222782135009766, "sampling/sampling_logp_difference/mean": 0.011874295771121979, "step": 1062, "step_time": 68.80869346344844 }, { "clip_ratio/high_max": 0.0008680555620230734, "clip_ratio/high_mean": 0.00017361111531499773, "clip_ratio/low_mean": 0.0004150390741415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0005886501894565299, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 824.0, "completions/mean_length": 942.7750244140625, "completions/mean_terminated_length": 663.0, "completions/min_length": 524.0, "completions/min_terminated_length": 524.0, "entropy": 0.07301829848438501, "epoch": 0.6046643913538111, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.008796866983175278, "kl": 0.008093779149930924, "learning_rate": 3.5631399317406146e-06, "loss": -0.0058, "num_tokens": 112406528.0, "reward": 0.45000001788139343, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.427361488342285, "sampling/importance_sampling_ratio/mean": 0.316293865442276, "sampling/importance_sampling_ratio/min": 0.008686468005180359, "sampling/sampling_logp_difference/max": 2.3187687397003174, "sampling/sampling_logp_difference/mean": 0.010202285833656788, "step": 1063, "step_time": 74.20525792776607 }, { "clip_ratio/high_max": 0.013909839908592403, "clip_ratio/high_mean": 0.003327417725813575, "clip_ratio/low_mean": 0.0014404297198780114, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004767847392940894, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 893.0, "completions/mean_length": 795.2250366210938, "completions/mean_terminated_length": 515.6111450195312, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.09473727736622095, "epoch": 0.6052332195676906, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018969278782606125, "kl": 0.012473016395233572, "learning_rate": 3.5580204778156996e-06, "loss": 0.0052, "num_tokens": 112499401.0, "reward": 0.800000011920929, "reward_std": 0.9391758441925049, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.490973711013794, "sampling/importance_sampling_ratio/mean": 0.5115882158279419, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.135258674621582, "sampling/sampling_logp_difference/mean": 0.011529594659805298, "step": 1064, "step_time": 71.1430707599502 }, { "clip_ratio/high_max": 0.013615464471513405, "clip_ratio/high_mean": 0.005225515058555175, "clip_ratio/low_mean": 0.0018798828314174898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00710539793362841, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 922.0, "completions/mean_length": 852.5250244140625, "completions/mean_terminated_length": 642.9444580078125, "completions/min_length": 423.0, "completions/min_terminated_length": 423.0, "entropy": 0.0776671888306737, "epoch": 0.60580204778157, "frac_reward_zero_std": 0.0, "grad_norm": 0.02273855172097683, "kl": 0.013478234526701272, "learning_rate": 3.552901023890785e-06, "loss": 0.0399, "num_tokens": 112603846.0, "reward": 0.8500000238418579, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.4961389899253845, "sampling/importance_sampling_ratio/max": 2.8322627544403076, "sampling/importance_sampling_ratio/mean": 0.38750016689300537, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.118821382522583, "sampling/sampling_logp_difference/mean": 0.011483109556138515, "step": 1065, "step_time": 71.87507856404409 }, { "clip_ratio/high_max": 0.019467013189569116, "clip_ratio/high_mean": 0.008203212230000645, "clip_ratio/low_mean": 0.0013183593982830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009521571337245405, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 757.9750366210938, "completions/mean_terminated_length": 614.7307739257812, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.12409343291074038, "epoch": 0.6063708759954494, "frac_reward_zero_std": 0.0, "grad_norm": 0.035662923008203506, "kl": 0.0237717175623402, "learning_rate": 3.54778156996587e-06, "loss": 0.0122, "num_tokens": 112710869.0, "reward": 0.9000000357627869, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.959630012512207, "sampling/importance_sampling_ratio/mean": 0.3406224846839905, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.088470220565796, "sampling/sampling_logp_difference/mean": 0.015188510529696941, "step": 1066, "step_time": 73.18256841925904 }, { "clip_ratio/high_max": 0.011227761278860271, "clip_ratio/high_mean": 0.002624802727950737, "clip_ratio/low_mean": 0.0022479989020212088, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004872801509918645, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 869.0, "completions/mean_length": 869.2000122070312, "completions/mean_terminated_length": 547.6923217773438, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.15148549154400826, "epoch": 0.6069397042093287, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0139564648270607, "kl": 0.014841762371361256, "learning_rate": 3.542662116040956e-06, "loss": -0.0113, "num_tokens": 112819117.0, "reward": 0.5250000357627869, "reward_std": 0.816103994846344, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.474740982055664, "sampling/importance_sampling_ratio/mean": 0.26930174231529236, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8946056365966797, "sampling/sampling_logp_difference/mean": 0.016707604750990868, "step": 1067, "step_time": 75.14282929361798 }, { "clip_ratio/high_max": 0.012174033792689443, "clip_ratio/high_mean": 0.003691891164635308, "clip_ratio/low_mean": 0.0010521666117710993, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004744057805510238, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 774.25, "completions/mean_terminated_length": 589.6521606445312, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.08458963967859745, "epoch": 0.6075085324232082, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018766213208436966, "kl": 0.012993561569601297, "learning_rate": 3.537542662116041e-06, "loss": 0.0243, "num_tokens": 112923599.0, "reward": 1.024999976158142, "reward_std": 0.9469925761222839, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.576555609703064, "sampling/importance_sampling_ratio/mean": 0.3058162331581116, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.316255569458008, "sampling/sampling_logp_difference/mean": 0.012232474982738495, "step": 1068, "step_time": 73.26427381066605 }, { "clip_ratio/high_max": 0.01172679360024631, "clip_ratio/high_mean": 0.003208593640010804, "clip_ratio/low_mean": 0.0030517577833961695, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006260351452510804, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 834.625, "completions/mean_terminated_length": 645.25, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.12561777420341969, "epoch": 0.6080773606370876, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026541979983448982, "kl": 0.012246634694747627, "learning_rate": 3.5324232081911264e-06, "loss": -0.0759, "num_tokens": 113027640.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9066665172576904, "sampling/importance_sampling_ratio/mean": 0.41943493485450745, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2292304039001465, "sampling/sampling_logp_difference/mean": 0.01496267318725586, "step": 1069, "step_time": 72.66305466508493 }, { "clip_ratio/high_max": 0.016784342005848885, "clip_ratio/high_mean": 0.005447253701277077, "clip_ratio/low_mean": 0.0025830615195445716, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008030314987991005, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 825.9500122070312, "completions/mean_terminated_length": 646.7619018554688, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.12339411024004221, "epoch": 0.608646188850967, "frac_reward_zero_std": 0.0, "grad_norm": 0.062211375683546066, "kl": 0.018174534663558006, "learning_rate": 3.5273037542662114e-06, "loss": 0.2566, "num_tokens": 113129030.0, "reward": 0.9750000238418579, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.6931943893432617, "sampling/importance_sampling_ratio/mean": 0.49377426505088806, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.443265199661255, "sampling/sampling_logp_difference/mean": 0.015023365616798401, "step": 1070, "step_time": 71.0571202179417 }, { "clip_ratio/high_max": 0.0170895226765424, "clip_ratio/high_mean": 0.00686500474694185, "clip_ratio/low_mean": 0.002235008156276308, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009100012888666242, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 761.0499877929688, "completions/mean_terminated_length": 585.75, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.10371766518801451, "epoch": 0.6092150170648464, "frac_reward_zero_std": 0.0, "grad_norm": 0.039619266986846924, "kl": 0.018832939909771085, "learning_rate": 3.5221843003412972e-06, "loss": -0.0862, "num_tokens": 113223712.0, "reward": 1.125, "reward_std": 0.9657599329948425, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.6538443565368652, "sampling/importance_sampling_ratio/mean": 0.43975314497947693, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.594892978668213, "sampling/sampling_logp_difference/mean": 0.014237465336918831, "step": 1071, "step_time": 71.60920750908554 }, { "clip_ratio/high_max": 0.012665657442994416, "clip_ratio/high_mean": 0.004243056115228683, "clip_ratio/low_mean": 0.003619045513914898, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.007862101658247411, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 970.0, "completions/mean_length": 845.2000122070312, "completions/mean_terminated_length": 577.0, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.14976980350911617, "epoch": 0.6097838452787259, "frac_reward_zero_std": 0.0, "grad_norm": 0.03380127623677254, "kl": 0.01205987180583179, "learning_rate": 3.5170648464163823e-06, "loss": 0.1185, "num_tokens": 113327952.0, "reward": 0.675000011920929, "reward_std": 0.888314425945282, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.9753323793411255, "sampling/importance_sampling_ratio/mean": 0.2834986746311188, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.960835576057434, "sampling/sampling_logp_difference/mean": 0.016171371564269066, "step": 1072, "step_time": 72.30680043576285 }, { "clip_ratio/high_max": 0.00953890843084082, "clip_ratio/high_mean": 0.002375516967731528, "clip_ratio/low_mean": 0.0009521484389551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033276653921348043, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 624.0, "completions/mean_length": 856.0750122070312, "completions/mean_terminated_length": 413.3636474609375, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.08613397600129247, "epoch": 0.6103526734926052, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.010209280997514725, "kl": 0.013000998995266855, "learning_rate": 3.5119453924914677e-06, "loss": 0.0158, "num_tokens": 113430315.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.817270040512085, "sampling/importance_sampling_ratio/mean": 0.4178386628627777, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0738353729248047, "sampling/sampling_logp_difference/mean": 0.010737104341387749, "step": 1073, "step_time": 70.0690793050453 }, { "clip_ratio/high_max": 0.016543691279366612, "clip_ratio/high_mean": 0.005966353084659204, "clip_ratio/low_mean": 0.0016357422136934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0076020953129045665, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 904.0, "completions/mean_length": 837.4750366210938, "completions/mean_terminated_length": 650.9500122070312, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "entropy": 0.13341592997312546, "epoch": 0.6109215017064846, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013600888662040234, "kl": 0.014267166494391859, "learning_rate": 3.5068259385665527e-06, "loss": -0.0259, "num_tokens": 113543182.0, "reward": 0.8500000238418579, "reward_std": 0.9212589263916016, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.166189432144165, "sampling/importance_sampling_ratio/mean": 0.309131383895874, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9752249717712402, "sampling/sampling_logp_difference/mean": 0.016173912212252617, "step": 1074, "step_time": 76.482387105003 }, { "clip_ratio/high_max": 0.014188467408530414, "clip_ratio/high_mean": 0.003334830660605803, "clip_ratio/low_mean": 0.0027343749679857865, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006069205701351166, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 975.0, "completions/mean_length": 948.2250366210938, "completions/mean_terminated_length": 687.2222290039062, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.1435262905433774, "epoch": 0.611490329920364, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01035996526479721, "kl": 0.01421396667137742, "learning_rate": 3.5017064846416386e-06, "loss": 0.0494, "num_tokens": 113655559.0, "reward": 0.42500001192092896, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290207743644714, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.117943048477173, "sampling/importance_sampling_ratio/mean": 0.23252539336681366, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.946638584136963, "sampling/sampling_logp_difference/mean": 0.01566912606358528, "step": 1075, "step_time": 75.38846961921081 }, { "clip_ratio/high_max": 0.010083621833473444, "clip_ratio/high_mean": 0.002915362478233874, "clip_ratio/low_mean": 0.001496358097938355, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004411720525240526, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 758.4249877929688, "completions/mean_terminated_length": 630.5555419921875, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.10594559833407402, "epoch": 0.6120591581342435, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.016020964831113815, "kl": 0.013683350873179734, "learning_rate": 3.4965870307167236e-06, "loss": 0.1028, "num_tokens": 113760608.0, "reward": 1.3250000476837158, "reward_std": 0.9442809820175171, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.960038185119629, "sampling/importance_sampling_ratio/mean": 0.5381250381469727, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4250967502593994, "sampling/sampling_logp_difference/mean": 0.014507449232041836, "step": 1076, "step_time": 73.5966382168699 }, { "clip_ratio/high_max": 0.013764289091341197, "clip_ratio/high_mean": 0.0036992581735830754, "clip_ratio/low_mean": 0.0019737062975764275, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005672964529367164, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 808.0, "completions/mean_length": 704.0, "completions/mean_terminated_length": 490.66668701171875, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.11643220018595457, "epoch": 0.6126279863481229, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01749584451317787, "kl": 0.015374067472293973, "learning_rate": 3.491467576791809e-06, "loss": 0.045, "num_tokens": 113868496.0, "reward": 0.949999988079071, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.6025829315185547, "sampling/importance_sampling_ratio/mean": 0.3936447501182556, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3524067401885986, "sampling/sampling_logp_difference/mean": 0.01579865626990795, "step": 1077, "step_time": 73.38387092272751 }, { "clip_ratio/high_max": 0.007716483742115088, "clip_ratio/high_mean": 0.0020469086448429152, "clip_ratio/low_mean": 0.0014404297035071068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003487338311970234, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 744.1500244140625, "completions/mean_terminated_length": 609.4074096679688, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.09739415533840656, "epoch": 0.6131968145620023, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018670061603188515, "kl": 0.012294595711864531, "learning_rate": 3.486348122866894e-06, "loss": -0.0205, "num_tokens": 113968254.0, "reward": 1.25, "reward_std": 0.9268086552619934, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.1946182250976562, "sampling/importance_sampling_ratio/mean": 0.38280588388442993, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2354519367218018, "sampling/sampling_logp_difference/mean": 0.013424227014183998, "step": 1078, "step_time": 71.83603224810213 }, { "clip_ratio/high_max": 0.006036265287548304, "clip_ratio/high_mean": 0.0015044298197608441, "clip_ratio/low_mean": 0.0021972656504658516, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037016954738646746, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 876.875, "completions/mean_terminated_length": 533.5833740234375, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.09347874857485294, "epoch": 0.6137656427758816, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01980750821530819, "kl": 0.012951933254953474, "learning_rate": 3.48122866894198e-06, "loss": 0.0282, "num_tokens": 114074753.0, "reward": 0.6000000238418579, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.7386606931686401, "sampling/importance_sampling_ratio/mean": 0.37870511412620544, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.453022003173828, "sampling/sampling_logp_difference/mean": 0.012621939182281494, "step": 1079, "step_time": 71.30697725620121 }, { "clip_ratio/high_max": 0.01655433070845902, "clip_ratio/high_mean": 0.006802564021199942, "clip_ratio/low_mean": 0.0025146484549622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00931721244705841, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 805.0, "completions/mean_length": 800.2250366210938, "completions/mean_terminated_length": 576.4500122070312, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 0.12969412747770548, "epoch": 0.6143344709897611, "frac_reward_zero_std": 0.0, "grad_norm": 0.02739783376455307, "kl": 0.016688739764504135, "learning_rate": 3.476109215017065e-06, "loss": 0.033, "num_tokens": 114184690.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.036320686340332, "sampling/importance_sampling_ratio/mean": 0.3131628632545471, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.556272029876709, "sampling/sampling_logp_difference/mean": 0.015602382831275463, "step": 1080, "step_time": 73.94480646587908 }, { "clip_ratio/high_max": 0.015210512792691588, "clip_ratio/high_mean": 0.006526775745442137, "clip_ratio/low_mean": 0.0010742187769210432, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007600994606036693, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 768.875, "completions/mean_terminated_length": 615.7999877929688, "completions/min_length": 432.0, "completions/min_terminated_length": 432.0, "entropy": 0.09553503803908825, "epoch": 0.6149032992036405, "frac_reward_zero_std": 0.0, "grad_norm": 0.14138375222682953, "kl": 0.012211047811433673, "learning_rate": 3.4709897610921503e-06, "loss": -0.0815, "num_tokens": 114291533.0, "reward": 1.25, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.0610454082489014, "sampling/importance_sampling_ratio/mean": 0.5039072036743164, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9643056392669678, "sampling/sampling_logp_difference/mean": 0.012747952714562416, "step": 1081, "step_time": 74.45704950205982 }, { "clip_ratio/high_max": 0.01576331432443112, "clip_ratio/high_mean": 0.005840739147970453, "clip_ratio/low_mean": 0.0021972656395519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008038004743866622, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 768.5, "completions/mean_terminated_length": 630.923095703125, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.1552603878080845, "epoch": 0.6154721274175199, "frac_reward_zero_std": 0.0, "grad_norm": 0.03120250254869461, "kl": 0.01664271205663681, "learning_rate": 3.4658703071672353e-06, "loss": 0.0119, "num_tokens": 114394193.0, "reward": 1.1750000715255737, "reward_std": 0.9306049942970276, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.9963856935501099, "sampling/importance_sampling_ratio/mean": 0.3096924424171448, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8501899242401123, "sampling/sampling_logp_difference/mean": 0.01687762327492237, "step": 1082, "step_time": 75.5291532932315 }, { "clip_ratio/high_max": 0.01321793021634221, "clip_ratio/high_mean": 0.004432665038621053, "clip_ratio/low_mean": 0.0009479674590693321, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0053806324722245336, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 921.0, "completions/mean_length": 718.7999877929688, "completions/mean_terminated_length": 535.6799926757812, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.11434629186987877, "epoch": 0.6160409556313993, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019699664786458015, "kl": 0.015926694264635444, "learning_rate": 3.460750853242321e-06, "loss": -0.0463, "num_tokens": 114497017.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.5404064655303955, "sampling/importance_sampling_ratio/mean": 0.43588757514953613, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.118821620941162, "sampling/sampling_logp_difference/mean": 0.01524913776665926, "step": 1083, "step_time": 74.09034450910985 }, { "clip_ratio/high_max": 0.008039330772589892, "clip_ratio/high_mean": 0.0020773293799720705, "clip_ratio/low_mean": 0.002197265625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004274595063179731, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 879.9249877929688, "completions/mean_terminated_length": 663.8125, "completions/min_length": 471.0, "completions/min_terminated_length": 471.0, "entropy": 0.0982478940859437, "epoch": 0.6166097838452788, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03442675992846489, "kl": 0.01056759082712233, "learning_rate": 3.455631399317406e-06, "loss": -0.1233, "num_tokens": 114599902.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.2264907360076904, "sampling/importance_sampling_ratio/mean": 0.5344123244285583, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.619469404220581, "sampling/sampling_logp_difference/mean": 0.012756923213601112, "step": 1084, "step_time": 70.69740453991108 }, { "clip_ratio/high_max": 0.009955017943866551, "clip_ratio/high_mean": 0.0023730490502202883, "clip_ratio/low_mean": 0.001513671872089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038867209368618205, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 807.0750122070312, "completions/mean_terminated_length": 541.9444580078125, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.06966326525434852, "epoch": 0.6171786120591581, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02228412963449955, "kl": 0.013156412984244525, "learning_rate": 3.4505119453924916e-06, "loss": 0.0511, "num_tokens": 114708137.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.7777013778686523, "sampling/importance_sampling_ratio/mean": 0.4528812766075134, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8377480506896973, "sampling/sampling_logp_difference/mean": 0.010343382135033607, "step": 1085, "step_time": 73.9032962268684 }, { "clip_ratio/high_max": 0.015544201654847711, "clip_ratio/high_mean": 0.004760377298225649, "clip_ratio/low_mean": 0.0015833540746825747, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006343731365632266, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 865.25, "completions/mean_terminated_length": 671.2222290039062, "completions/min_length": 460.0, "completions/min_terminated_length": 460.0, "entropy": 0.14490450080484152, "epoch": 0.6177474402730375, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024419564753770828, "kl": 0.013362093013711274, "learning_rate": 3.4453924914675766e-06, "loss": -0.0349, "num_tokens": 114820147.0, "reward": 0.75, "reward_std": 0.8987170457839966, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.685739278793335, "sampling/importance_sampling_ratio/mean": 0.48449087142944336, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.127293109893799, "sampling/sampling_logp_difference/mean": 0.015727141872048378, "step": 1086, "step_time": 74.4418905491475 }, { "clip_ratio/high_max": 0.015180722693912685, "clip_ratio/high_mean": 0.005652650346746668, "clip_ratio/low_mean": 0.0017556132879690267, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007408263743855059, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 836.2000122070312, "completions/mean_terminated_length": 697.3912963867188, "completions/min_length": 438.0, "completions/min_terminated_length": 438.0, "entropy": 0.1345727313309908, "epoch": 0.6183162684869169, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023257259279489517, "kl": 0.013332078466191888, "learning_rate": 3.4402730375426625e-06, "loss": 0.1065, "num_tokens": 114929219.0, "reward": 1.0500000715255737, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.5873539447784424, "sampling/importance_sampling_ratio/mean": 0.2783942222595215, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9600645303726196, "sampling/sampling_logp_difference/mean": 0.01615673489868641, "step": 1087, "step_time": 73.91175867500715 }, { "clip_ratio/high_max": 0.008073824923485518, "clip_ratio/high_mean": 0.0018160531908506528, "clip_ratio/low_mean": 0.0015604082163918065, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003376461419975385, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 941.0, "completions/mean_length": 741.375, "completions/mean_terminated_length": 571.7999877929688, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.0868205763399601, "epoch": 0.6188850967007964, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.030251454561948776, "kl": 0.01535401598084718, "learning_rate": 3.4351535836177475e-06, "loss": -0.0117, "num_tokens": 115017346.0, "reward": 0.9750000238418579, "reward_std": 0.8619447350502014, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.7223241329193115, "sampling/importance_sampling_ratio/mean": 0.5446793437004089, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4494247436523438, "sampling/sampling_logp_difference/mean": 0.013293038122355938, "step": 1088, "step_time": 73.13827521400526 }, { "clip_ratio/high_max": 0.014132204931229353, "clip_ratio/high_mean": 0.0029884280229452997, "clip_ratio/low_mean": 0.0018712957680691034, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0048597238783258945, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 886.2999877929688, "completions/mean_terminated_length": 600.3077392578125, "completions/min_length": 402.0, "completions/min_terminated_length": 402.0, "entropy": 0.08746102266013622, "epoch": 0.6194539249146758, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.019368695095181465, "kl": 0.008893261838238686, "learning_rate": 3.430034129692833e-06, "loss": -0.0509, "num_tokens": 115121758.0, "reward": 0.5, "reward_std": 0.7844645380973816, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.9410488605499268, "sampling/importance_sampling_ratio/mean": 0.502236008644104, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2553112506866455, "sampling/sampling_logp_difference/mean": 0.011653106659650803, "step": 1089, "step_time": 72.8318678769283 }, { "clip_ratio/high_max": 0.010976771940477192, "clip_ratio/high_mean": 0.002445855403493624, "clip_ratio/low_mean": 0.0013183593982830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037642146926373243, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 653.0, "completions/mean_length": 889.25, "completions/mean_terminated_length": 425.1111145019531, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.1093570701777935, "epoch": 0.6200227531285551, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02562505565583706, "kl": 0.009335098089650273, "learning_rate": 3.4249146757679184e-06, "loss": -0.029, "num_tokens": 115223792.0, "reward": 0.45000001788139343, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290207743644714, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.421638250350952, "sampling/importance_sampling_ratio/mean": 0.3660328984260559, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.433779239654541, "sampling/sampling_logp_difference/mean": 0.012408862821757793, "step": 1090, "step_time": 71.28814661549404 }, { "clip_ratio/high_max": 0.010033157595898956, "clip_ratio/high_mean": 0.003200512561306823, "clip_ratio/low_mean": 0.0030775227351114154, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.006278035289142281, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 960.0, "completions/mean_length": 898.3500366210938, "completions/mean_terminated_length": 665.0, "completions/min_length": 468.0, "completions/min_terminated_length": 468.0, "entropy": 0.1153783043846488, "epoch": 0.6205915813424346, "frac_reward_zero_std": 0.0, "grad_norm": 0.03396408259868622, "kl": 0.016582478070631623, "learning_rate": 3.4197952218430034e-06, "loss": 0.1377, "num_tokens": 115336870.0, "reward": 0.6500000357627869, "reward_std": 0.9212588667869568, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.171647071838379, "sampling/importance_sampling_ratio/mean": 0.3920711874961853, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6329033374786377, "sampling/sampling_logp_difference/mean": 0.013999024406075478, "step": 1091, "step_time": 80.2031133172568 }, { "clip_ratio/high_max": 0.011268006172031164, "clip_ratio/high_mean": 0.0037210601731203496, "clip_ratio/low_mean": 0.0008056640726863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004526724194874987, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 867.0, "completions/mean_length": 590.4750366210938, "completions/mean_terminated_length": 464.6128845214844, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.10930857714265585, "epoch": 0.621160409556314, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025048857554793358, "kl": 0.01451495650690049, "learning_rate": 3.414675767918089e-06, "loss": 0.0686, "num_tokens": 115436001.0, "reward": 1.5500000715255737, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.7749999761581421, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 1.8154493570327759, "sampling/importance_sampling_ratio/mean": 0.48657265305519104, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9860289096832275, "sampling/sampling_logp_difference/mean": 0.014447236433625221, "step": 1092, "step_time": 67.85442923707888 }, { "clip_ratio/high_max": 0.01504125352948904, "clip_ratio/high_mean": 0.004429171764058992, "clip_ratio/low_mean": 0.0016601562674622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006089327915105969, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 870.0, "completions/mean_length": 837.2000122070312, "completions/mean_terminated_length": 525.86669921875, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.11413159128278494, "epoch": 0.6217292377701934, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03839990124106407, "kl": 0.01746600412297994, "learning_rate": 3.409556313993174e-06, "loss": -0.0957, "num_tokens": 115540185.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.7655651569366455, "sampling/importance_sampling_ratio/mean": 0.5260424017906189, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9425945281982422, "sampling/sampling_logp_difference/mean": 0.014223671518266201, "step": 1093, "step_time": 71.1452881586738 }, { "clip_ratio/high_max": 0.011208612937480211, "clip_ratio/high_mean": 0.0028933721914654598, "clip_ratio/low_mean": 0.0011474609291326487, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004040833155158907, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 967.0, "completions/mean_length": 708.9750366210938, "completions/mean_terminated_length": 539.34619140625, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.10102805867791176, "epoch": 0.6222980659840728, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02152807079255581, "kl": 0.011648669023998082, "learning_rate": 3.4044368600682597e-06, "loss": -0.0318, "num_tokens": 115640712.0, "reward": 1.3000000715255737, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.3412957191467285, "sampling/importance_sampling_ratio/mean": 0.5639311075210571, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.33602237701416, "sampling/sampling_logp_difference/mean": 0.014383438043296337, "step": 1094, "step_time": 72.20975656225346 }, { "clip_ratio/high_max": 0.01722524012438953, "clip_ratio/high_mean": 0.00732676760526374, "clip_ratio/low_mean": 0.0009521484389551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008278916007839143, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 683.7250366210938, "completions/mean_terminated_length": 584.9354858398438, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.1208778340369463, "epoch": 0.6228668941979523, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01719166524708271, "kl": 0.014898406341671944, "learning_rate": 3.3993174061433447e-06, "loss": -0.0983, "num_tokens": 115750509.0, "reward": 1.5500000715255737, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.7749999761581421, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.383028268814087, "sampling/importance_sampling_ratio/mean": 0.3271036744117737, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2900867462158203, "sampling/sampling_logp_difference/mean": 0.015161479823291302, "step": 1095, "step_time": 72.67960782465525 }, { "clip_ratio/high_max": 0.00733524962561205, "clip_ratio/high_mean": 0.002215163618529914, "clip_ratio/low_mean": 0.0029296875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005144850962096825, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 819.3250122070312, "completions/mean_terminated_length": 542.4117431640625, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.12934093549847603, "epoch": 0.6234357224118316, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01717003993690014, "kl": 0.01255685812793672, "learning_rate": 3.39419795221843e-06, "loss": -0.0223, "num_tokens": 115859090.0, "reward": 0.550000011920929, "reward_std": 0.7143223285675049, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 1.7503019571304321, "sampling/importance_sampling_ratio/mean": 0.2637736201286316, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.862513780593872, "sampling/sampling_logp_difference/mean": 0.015596416778862476, "step": 1096, "step_time": 74.68410134897567 }, { "clip_ratio/high_max": 0.015564693370833993, "clip_ratio/high_mean": 0.0054307394893839955, "clip_ratio/low_mean": 0.0022705078590661287, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007701247115619481, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 856.0750122070312, "completions/mean_terminated_length": 670.4736938476562, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.12931507732719183, "epoch": 0.624004550625711, "frac_reward_zero_std": 0.0, "grad_norm": 0.02904334105551243, "kl": 0.01879754662513733, "learning_rate": 3.389078498293515e-06, "loss": 0.0959, "num_tokens": 115967389.0, "reward": 0.875, "reward_std": 0.9657599329948425, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.6379950046539307, "sampling/importance_sampling_ratio/mean": 0.4233631193637848, "sampling/importance_sampling_ratio/min": 0.0010942500084638596, "sampling/sampling_logp_difference/max": 2.3549792766571045, "sampling/sampling_logp_difference/mean": 0.014659670181572437, "step": 1097, "step_time": 76.50189658533782 }, { "clip_ratio/high_max": 0.014856296766083688, "clip_ratio/high_mean": 0.005445701535791159, "clip_ratio/low_mean": 0.0020996094099245965, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007545310945715755, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 795.6749877929688, "completions/mean_terminated_length": 672.7307739257812, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.17476117424666882, "epoch": 0.6245733788395904, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020272592082619667, "kl": 0.01783896773122251, "learning_rate": 3.383959044368601e-06, "loss": -0.0318, "num_tokens": 116080656.0, "reward": 1.149999976158142, "reward_std": 0.9212588667869568, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.460444927215576, "sampling/importance_sampling_ratio/mean": 0.3628217279911041, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.120117664337158, "sampling/sampling_logp_difference/mean": 0.019665582105517387, "step": 1098, "step_time": 74.86298699211329 }, { "clip_ratio/high_max": 0.01029066531918943, "clip_ratio/high_mean": 0.0034027365763904527, "clip_ratio/low_mean": 0.0009521484626020538, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004354884964413941, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 794.0, "completions/mean_length": 817.0750122070312, "completions/mean_terminated_length": 472.20001220703125, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.1294784126803279, "epoch": 0.6251422070534699, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0395900197327137, "kl": 0.011255994555540383, "learning_rate": 3.378839590443686e-06, "loss": -0.1099, "num_tokens": 116189539.0, "reward": 0.5250000357627869, "reward_std": 0.7506407499313354, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.628862142562866, "sampling/importance_sampling_ratio/mean": 0.38793298602104187, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4738311767578125, "sampling/sampling_logp_difference/mean": 0.014842662028968334, "step": 1099, "step_time": 74.22136768768542 }, { "clip_ratio/high_max": 0.015201347356196493, "clip_ratio/high_mean": 0.003255323215853423, "clip_ratio/low_mean": 0.0010253906457364792, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004280713910702616, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 787.6500244140625, "completions/mean_terminated_length": 433.125, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.11177398636937141, "epoch": 0.6257110352673493, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009612212888896465, "kl": 0.01021387369837612, "learning_rate": 3.3737201365187714e-06, "loss": 0.0222, "num_tokens": 116290333.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.0729825496673584, "sampling/importance_sampling_ratio/mean": 0.49475377798080444, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8956146240234375, "sampling/sampling_logp_difference/mean": 0.013184311799705029, "step": 1100, "step_time": 69.65981562738307 }, { "clip_ratio/high_max": 0.013846290938090533, "clip_ratio/high_mean": 0.0031816371774766594, "clip_ratio/low_mean": 0.0005859375123691279, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037675746716558933, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 652.0, "completions/mean_length": 772.7750244140625, "completions/mean_terminated_length": 465.72222900390625, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.1185380769893527, "epoch": 0.6262798634812287, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01969284564256668, "kl": 0.018015044159255922, "learning_rate": 3.3686006825938565e-06, "loss": -0.0645, "num_tokens": 116398644.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.932819366455078, "sampling/importance_sampling_ratio/mean": 0.4877108633518219, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5795831680297852, "sampling/sampling_logp_difference/mean": 0.014761090278625488, "step": 1101, "step_time": 73.53501695115119 }, { "clip_ratio/high_max": 0.011618116463068873, "clip_ratio/high_mean": 0.00385735476447735, "clip_ratio/low_mean": 0.0013916015577706276, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005248956236755475, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 781.9000244140625, "completions/mean_terminated_length": 514.3157958984375, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.09140170272439718, "epoch": 0.626848691695108, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01986689493060112, "kl": 0.011449927696958184, "learning_rate": 3.3634812286689423e-06, "loss": -0.0111, "num_tokens": 116509552.0, "reward": 0.824999988079071, "reward_std": 0.9306049942970276, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.75699782371521, "sampling/importance_sampling_ratio/mean": 0.5344257354736328, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6574900150299072, "sampling/sampling_logp_difference/mean": 0.013052001595497131, "step": 1102, "step_time": 74.26296764053404 }, { "clip_ratio/high_max": 0.010744280007202178, "clip_ratio/high_mean": 0.0032720881863497198, "clip_ratio/low_mean": 0.0018066406519210432, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005078728863736615, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 905.75, "completions/mean_terminated_length": 686.1428833007812, "completions/min_length": 428.0, "completions/min_terminated_length": 428.0, "entropy": 0.1081985579803586, "epoch": 0.6274175199089875, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02818906493484974, "kl": 0.012255495530553162, "learning_rate": 3.3583617747440273e-06, "loss": -0.0688, "num_tokens": 116623062.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.764682650566101, "sampling/importance_sampling_ratio/mean": 0.34578484296798706, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1547305583953857, "sampling/sampling_logp_difference/mean": 0.013203481212258339, "step": 1103, "step_time": 74.8932011644356 }, { "clip_ratio/high_max": 0.006384822161635384, "clip_ratio/high_mean": 0.001580454561917577, "clip_ratio/low_mean": 0.0007370474650087999, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002317502032383345, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 808.1500244140625, "completions/mean_terminated_length": 516.11767578125, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.1372930221259594, "epoch": 0.6279863481228669, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.021167941391468048, "kl": 0.01119784137699753, "learning_rate": 3.3532423208191128e-06, "loss": -0.0035, "num_tokens": 116733060.0, "reward": 0.7750000357627869, "reward_std": 0.9469925761222839, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.5856564044952393, "sampling/importance_sampling_ratio/mean": 0.43431609869003296, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.641676425933838, "sampling/sampling_logp_difference/mean": 0.015256228856742382, "step": 1104, "step_time": 74.73113099299371 }, { "clip_ratio/high_max": 0.011976617854088545, "clip_ratio/high_mean": 0.003861435456201434, "clip_ratio/low_mean": 0.0029052734171273187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0067667089169844985, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 825.0, "completions/mean_length": 805.7750244140625, "completions/mean_terminated_length": 510.5294189453125, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.12089674640446901, "epoch": 0.6285551763367463, "frac_reward_zero_std": 0.0, "grad_norm": 0.015146628953516483, "kl": 0.0137559634167701, "learning_rate": 3.3481228668941978e-06, "loss": 0.0548, "num_tokens": 116835203.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.7565743923187256, "sampling/importance_sampling_ratio/mean": 0.40823832154273987, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9737048149108887, "sampling/sampling_logp_difference/mean": 0.013646036386489868, "step": 1105, "step_time": 74.6143652482424 }, { "clip_ratio/high_max": 0.014193023322150111, "clip_ratio/high_mean": 0.0042620115127647296, "clip_ratio/low_mean": 0.0024902343284338713, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006752245797542855, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 771.0, "completions/mean_length": 880.5750122070312, "completions/mean_terminated_length": 582.6923217773438, "completions/min_length": 416.0, "completions/min_terminated_length": 416.0, "entropy": 0.12422810029238462, "epoch": 0.6291240045506257, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020407434552907944, "kl": 0.019181916140951216, "learning_rate": 3.3430034129692836e-06, "loss": 0.0231, "num_tokens": 116942890.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.9957616329193115, "sampling/importance_sampling_ratio/mean": 0.2825942635536194, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3037497997283936, "sampling/sampling_logp_difference/mean": 0.015136703848838806, "step": 1106, "step_time": 74.15560389822349 }, { "clip_ratio/high_max": 0.015374303329735994, "clip_ratio/high_mean": 0.007509422051953152, "clip_ratio/low_mean": 0.0011962890857830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008705711166840047, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 939.0, "completions/mean_length": 773.0499877929688, "completions/mean_terminated_length": 652.2222290039062, "completions/min_length": 420.0, "completions/min_terminated_length": 420.0, "entropy": 0.09964283369481564, "epoch": 0.6296928327645052, "frac_reward_zero_std": 0.0, "grad_norm": 0.022215252742171288, "kl": 0.010761283338069916, "learning_rate": 3.3378839590443686e-06, "loss": 0.025, "num_tokens": 117040012.0, "reward": 1.350000023841858, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.409716844558716, "sampling/importance_sampling_ratio/mean": 0.5248020887374878, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9074819087982178, "sampling/sampling_logp_difference/mean": 0.012737303040921688, "step": 1107, "step_time": 71.86076328135096 }, { "clip_ratio/high_max": 0.01595423580147326, "clip_ratio/high_mean": 0.004395410243887454, "clip_ratio/low_mean": 0.003442327244556509, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00783773756120354, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 853.8500366210938, "completions/mean_terminated_length": 699.90478515625, "completions/min_length": 402.0, "completions/min_terminated_length": 402.0, "entropy": 0.11097644176334143, "epoch": 0.6302616609783845, "frac_reward_zero_std": 0.0, "grad_norm": 0.02426457218825817, "kl": 0.011078628012910485, "learning_rate": 3.332764505119454e-06, "loss": -0.0629, "num_tokens": 117137982.0, "reward": 0.8500000238418579, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.0629143714904785, "sampling/importance_sampling_ratio/mean": 0.379005491733551, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.825745105743408, "sampling/sampling_logp_difference/mean": 0.013061332516372204, "step": 1108, "step_time": 72.76905059767887 }, { "clip_ratio/high_max": 0.011275716999080032, "clip_ratio/high_mean": 0.003849903107038699, "clip_ratio/low_mean": 0.0009033203023136593, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004753223431180231, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 623.4000244140625, "completions/mean_terminated_length": 552.7058715820312, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.11384484264999628, "epoch": 0.6308304891922639, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019577259197831154, "kl": 0.014352145604789257, "learning_rate": 3.327645051194539e-06, "loss": -0.0757, "num_tokens": 117225326.0, "reward": 1.5, "reward_std": 0.751067578792572, "rewards/format_reward/mean": 0.8500000238418579, "rewards/format_reward/std": 0.3616202771663666, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.3016183376312256, "sampling/importance_sampling_ratio/mean": 0.4225197434425354, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.555549383163452, "sampling/sampling_logp_difference/mean": 0.015990903601050377, "step": 1109, "step_time": 69.64910237467848 }, { "clip_ratio/high_max": 0.010993499250616878, "clip_ratio/high_mean": 0.0035029380960622802, "clip_ratio/low_mean": 0.000610351571594947, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00411328962945845, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 939.0, "completions/mean_length": 877.9249877929688, "completions/mean_terminated_length": 634.4666748046875, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.09850211534649134, "epoch": 0.6313993174061433, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019118519499897957, "kl": 0.010628767078742385, "learning_rate": 3.322525597269625e-06, "loss": 0.039, "num_tokens": 117328411.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.124612331390381, "sampling/importance_sampling_ratio/mean": 0.4903218150138855, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.291595220565796, "sampling/sampling_logp_difference/mean": 0.012087284587323666, "step": 1110, "step_time": 75.50932983169332 }, { "clip_ratio/high_max": 0.01622244733152911, "clip_ratio/high_mean": 0.006843838054919615, "clip_ratio/low_mean": 0.001587009959621355, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00843084801454097, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 606.9750366210938, "completions/mean_terminated_length": 448.7930908203125, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.09523204993456602, "epoch": 0.6319681456200228, "frac_reward_zero_std": 0.0, "grad_norm": 0.03193415328860283, "kl": 0.014295975910499692, "learning_rate": 3.31740614334471e-06, "loss": -0.1281, "num_tokens": 117413130.0, "reward": 1.375, "reward_std": 0.896932065486908, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.626753330230713, "sampling/importance_sampling_ratio/mean": 0.5086143612861633, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8982272148132324, "sampling/sampling_logp_difference/mean": 0.012938079424202442, "step": 1111, "step_time": 73.61358104017563 }, { "clip_ratio/high_max": 0.013476109714247286, "clip_ratio/high_mean": 0.004888326162472367, "clip_ratio/low_mean": 0.0015726838282716926, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006461010023485869, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 839.5, "completions/mean_terminated_length": 562.75, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.11510247364640236, "epoch": 0.6325369738339022, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029425740242004395, "kl": 0.014805021462962031, "learning_rate": 3.3122866894197954e-06, "loss": -0.0752, "num_tokens": 117521198.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.8495917320251465, "sampling/importance_sampling_ratio/mean": 0.25241005420684814, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9804158210754395, "sampling/sampling_logp_difference/mean": 0.014404348097741604, "step": 1112, "step_time": 74.78503706655465 }, { "clip_ratio/high_max": 0.016163588734343648, "clip_ratio/high_mean": 0.006425868778023869, "clip_ratio/low_mean": 0.0013267482827359345, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007752617006190121, "completions/clipped_ratio": 0.17499999701976776, "completions/max_length": 1024.0, "completions/max_terminated_length": 914.0, "completions/mean_length": 571.0250244140625, "completions/mean_terminated_length": 474.9394226074219, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.10152194648981094, "epoch": 0.6331058020477816, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.08646813780069351, "kl": 0.01564497477374971, "learning_rate": 3.3071672354948804e-06, "loss": -0.0805, "num_tokens": 117613351.0, "reward": 1.524999976158142, "reward_std": 0.816103994846344, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.7250000238418579, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.675335645675659, "sampling/importance_sampling_ratio/mean": 0.6111881136894226, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.687602996826172, "sampling/sampling_logp_difference/mean": 0.01525917835533619, "step": 1113, "step_time": 70.57735486817546 }, { "clip_ratio/high_max": 0.016644430172163993, "clip_ratio/high_mean": 0.006412382121197879, "clip_ratio/low_mean": 0.002148437502910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008560819434933364, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 769.25, "completions/mean_terminated_length": 538.7619018554688, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.144791129976511, "epoch": 0.6336746302616609, "frac_reward_zero_std": 0.0, "grad_norm": 0.032658763229846954, "kl": 0.016205001971684396, "learning_rate": 3.3020477815699662e-06, "loss": -0.1141, "num_tokens": 117714385.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.4179084300994873, "sampling/importance_sampling_ratio/mean": 0.4136505126953125, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3113369941711426, "sampling/sampling_logp_difference/mean": 0.016070319339632988, "step": 1114, "step_time": 73.99740868411027 }, { "clip_ratio/high_max": 0.010166348598431796, "clip_ratio/high_mean": 0.0033306375262327492, "clip_ratio/low_mean": 0.00117187501200533, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004502512398175895, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 952.0, "completions/mean_length": 836.4500122070312, "completions/mean_terminated_length": 666.7619018554688, "completions/min_length": 477.0, "completions/min_terminated_length": 477.0, "entropy": 0.15033577755093575, "epoch": 0.6342434584755404, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011620079167187214, "kl": 0.012865268159657717, "learning_rate": 3.2969283276450513e-06, "loss": -0.0049, "num_tokens": 117819667.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.412588596343994, "sampling/importance_sampling_ratio/mean": 0.3653637766838074, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.074455738067627, "sampling/sampling_logp_difference/mean": 0.01680414192378521, "step": 1115, "step_time": 72.58884780481458 }, { "clip_ratio/high_max": 0.013450412079691887, "clip_ratio/high_mean": 0.00509423715993762, "clip_ratio/low_mean": 0.0014160156351863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006510252831503749, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 765.1749877929688, "completions/mean_terminated_length": 609.8800048828125, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.09472066722810268, "epoch": 0.6348122866894198, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013278602622449398, "kl": 0.019823683775030077, "learning_rate": 3.2918088737201367e-06, "loss": -0.0318, "num_tokens": 117920962.0, "reward": 1.25, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.4505527019500732, "sampling/importance_sampling_ratio/mean": 0.3690944015979767, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0571961402893066, "sampling/sampling_logp_difference/mean": 0.013075794093310833, "step": 1116, "step_time": 77.62043702788651 }, { "clip_ratio/high_max": 0.01526419899892062, "clip_ratio/high_mean": 0.006757640279829502, "clip_ratio/low_mean": 0.0018066406264551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008564280869904906, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 797.6000366210938, "completions/mean_terminated_length": 630.2608642578125, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.11208016332238913, "epoch": 0.6353811149032992, "frac_reward_zero_std": 0.0, "grad_norm": 0.044554151594638824, "kl": 0.015875707264058292, "learning_rate": 3.2866894197952217e-06, "loss": 0.0536, "num_tokens": 118029618.0, "reward": 0.9000000357627869, "reward_std": 0.8711913824081421, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.5202399492263794, "sampling/importance_sampling_ratio/mean": 0.3902312219142914, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.317091941833496, "sampling/sampling_logp_difference/mean": 0.014719739556312561, "step": 1117, "step_time": 75.13827941566706 }, { "clip_ratio/high_max": 0.021080964128486812, "clip_ratio/high_mean": 0.006378400459652767, "clip_ratio/low_mean": 0.0011230468680878403, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007501447224058211, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 786.0, "completions/mean_length": 807.4000244140625, "completions/mean_terminated_length": 542.6666870117188, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.07227912452071905, "epoch": 0.6359499431171786, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01431175135076046, "kl": 0.011025446583516896, "learning_rate": 3.2815699658703076e-06, "loss": -0.0369, "num_tokens": 118130994.0, "reward": 0.75, "reward_std": 0.8987169861793518, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.78193736076355, "sampling/importance_sampling_ratio/mean": 0.3376486897468567, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0171396732330322, "sampling/sampling_logp_difference/mean": 0.01068786159157753, "step": 1118, "step_time": 71.3644320575986 }, { "clip_ratio/high_max": 0.015409291780088097, "clip_ratio/high_mean": 0.005677392749930732, "clip_ratio/low_mean": 0.001879882827779511, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00755727564683184, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 856.2000122070312, "completions/mean_terminated_length": 651.1111450195312, "completions/min_length": 460.0, "completions/min_terminated_length": 460.0, "entropy": 0.1365661434829235, "epoch": 0.636518771331058, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04745185002684593, "kl": 0.010658324346877635, "learning_rate": 3.2764505119453926e-06, "loss": 0.2435, "num_tokens": 118239466.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.9819819927215576, "sampling/importance_sampling_ratio/mean": 0.4454285502433777, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3471320867538452, "sampling/sampling_logp_difference/mean": 0.015565655194222927, "step": 1119, "step_time": 75.68789799953811 }, { "clip_ratio/high_max": 0.012126773886848241, "clip_ratio/high_mean": 0.0030417095113079995, "clip_ratio/low_mean": 0.001708984375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004750693798996508, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 820.0, "completions/mean_length": 798.375, "completions/mean_terminated_length": 522.6111450195312, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.11015340220183134, "epoch": 0.6370875995449374, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014390461146831512, "kl": 0.01307256449945271, "learning_rate": 3.271331058020478e-06, "loss": 0.0445, "num_tokens": 118351537.0, "reward": 0.9000000357627869, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.7885093688964844, "sampling/importance_sampling_ratio/mean": 0.37773600220680237, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9187233448028564, "sampling/sampling_logp_difference/mean": 0.014064264483749866, "step": 1120, "step_time": 76.52371028973721 }, { "clip_ratio/high_max": 0.010389501403551549, "clip_ratio/high_mean": 0.002380145582719706, "clip_ratio/low_mean": 0.0023437499621650204, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004723895428469405, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 967.5, "completions/mean_terminated_length": 701.1428833007812, "completions/min_length": 542.0, "completions/min_terminated_length": 542.0, "entropy": 0.11100248526781797, "epoch": 0.6376564277588168, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014267115853726864, "kl": 0.01102365500992164, "learning_rate": 3.266211604095563e-06, "loss": 0.0148, "num_tokens": 118464453.0, "reward": 0.2750000059604645, "reward_std": 0.6400120258331299, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.676908493041992, "sampling/importance_sampling_ratio/mean": 0.5832678079605103, "sampling/importance_sampling_ratio/min": 0.0016966034891083837, "sampling/sampling_logp_difference/max": 2.4189884662628174, "sampling/sampling_logp_difference/mean": 0.012350707314908504, "step": 1121, "step_time": 76.3451989251189 }, { "clip_ratio/high_max": 0.01002572278957814, "clip_ratio/high_mean": 0.0028380438743624836, "clip_ratio/low_mean": 0.002197265584982233, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005035309499362484, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 972.0250244140625, "completions/mean_terminated_length": 864.0769653320312, "completions/min_length": 625.0, "completions/min_terminated_length": 625.0, "entropy": 0.12228438723832369, "epoch": 0.6382252559726962, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024655967950820923, "kl": 0.011424366384744644, "learning_rate": 3.261092150170649e-06, "loss": -0.0288, "num_tokens": 118580398.0, "reward": 0.574999988079071, "reward_std": 0.8737629055976868, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.224238395690918, "sampling/importance_sampling_ratio/mean": 0.3844344913959503, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.821725845336914, "sampling/sampling_logp_difference/mean": 0.013327701948583126, "step": 1122, "step_time": 77.87016947800294 }, { "clip_ratio/high_max": 0.01527524646371603, "clip_ratio/high_mean": 0.006123384780948982, "clip_ratio/low_mean": 0.0010558790218055947, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007179263804573566, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 635.875, "completions/mean_terminated_length": 538.84375, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.10324501153081656, "epoch": 0.6387940841865757, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024477878585457802, "kl": 0.015953921945765615, "learning_rate": 3.255972696245734e-06, "loss": -0.1135, "num_tokens": 118681145.0, "reward": 1.5750000476837158, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.7749999761581421, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.637402057647705, "sampling/importance_sampling_ratio/mean": 0.4018469750881195, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.005537509918213, "sampling/sampling_logp_difference/mean": 0.014682984910905361, "step": 1123, "step_time": 71.1856809060555 }, { "clip_ratio/high_max": 0.015190442791208625, "clip_ratio/high_mean": 0.006111267604865134, "clip_ratio/low_mean": 0.002377769342274405, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008489037049002945, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 747.9000244140625, "completions/mean_terminated_length": 543.8261108398438, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.1311946026980877, "epoch": 0.6393629124004551, "frac_reward_zero_std": 0.0, "grad_norm": 0.033602599054574966, "kl": 0.01763622893486172, "learning_rate": 3.2508532423208193e-06, "loss": 0.0788, "num_tokens": 118782125.0, "reward": 1.0750000476837158, "reward_std": 0.9710555076599121, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.354116201400757, "sampling/importance_sampling_ratio/mean": 0.265406996011734, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7048871517181396, "sampling/sampling_logp_difference/mean": 0.01588287577033043, "step": 1124, "step_time": 75.2955282353796 }, { "clip_ratio/high_max": 0.008727658772841096, "clip_ratio/high_mean": 0.0025227661099052057, "clip_ratio/low_mean": 0.0008544921856810106, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003377258326509036, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 682.7750244140625, "completions/mean_terminated_length": 536.5357666015625, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.07034684624522924, "epoch": 0.6399317406143344, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.036328691989183426, "kl": 0.011220880900509655, "learning_rate": 3.2457337883959043e-06, "loss": -0.1884, "num_tokens": 118884212.0, "reward": 1.399999976158142, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.699999988079071, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.9889628887176514, "sampling/importance_sampling_ratio/mean": 0.5715748071670532, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8857353925704956, "sampling/sampling_logp_difference/mean": 0.011367544531822205, "step": 1125, "step_time": 72.01924322196282 }, { "clip_ratio/high_max": 0.01526893232949078, "clip_ratio/high_mean": 0.0040641156665515155, "clip_ratio/low_mean": 0.001611328130820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005675443739164621, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 889.0, "completions/mean_length": 886.6000366210938, "completions/mean_terminated_length": 601.2307739257812, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "entropy": 0.08135220175608993, "epoch": 0.6405005688282139, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020258188247680664, "kl": 0.009255671873688698, "learning_rate": 3.24061433447099e-06, "loss": 0.0518, "num_tokens": 118990060.0, "reward": 0.5250000357627869, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.8618474006652832, "sampling/importance_sampling_ratio/mean": 0.4035053253173828, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.77028226852417, "sampling/sampling_logp_difference/mean": 0.010665926150977612, "step": 1126, "step_time": 72.0997879030183 }, { "clip_ratio/high_max": 0.0131300357170403, "clip_ratio/high_mean": 0.0048465540166944265, "clip_ratio/low_mean": 0.0021380569742177613, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006984611216466874, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 942.0, "completions/mean_length": 757.1000366210938, "completions/mean_terminated_length": 596.9599609375, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.13115183170884848, "epoch": 0.6410693970420933, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03952344134449959, "kl": 0.027263222262263298, "learning_rate": 3.235494880546075e-06, "loss": 0.04, "num_tokens": 119090048.0, "reward": 1.149999976158142, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.5240278244018555, "sampling/importance_sampling_ratio/mean": 0.4995952248573303, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4835591316223145, "sampling/sampling_logp_difference/mean": 0.015811458230018616, "step": 1127, "step_time": 71.22911233338527 }, { "clip_ratio/high_max": 0.013656030874699354, "clip_ratio/high_mean": 0.004662159954023082, "clip_ratio/low_mean": 0.001991722143429797, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006653882097452879, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.0, "completions/mean_length": 843.2999877929688, "completions/mean_terminated_length": 643.5789794921875, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.11179618909955025, "epoch": 0.6416382252559727, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018753619864583015, "kl": 0.013308165362104774, "learning_rate": 3.23037542662116e-06, "loss": -0.0966, "num_tokens": 119194876.0, "reward": 0.875, "reward_std": 0.9657599925994873, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.211174964904785, "sampling/importance_sampling_ratio/mean": 0.4419359266757965, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.388727903366089, "sampling/sampling_logp_difference/mean": 0.013259807601571083, "step": 1128, "step_time": 71.12826267909259 }, { "clip_ratio/high_max": 0.01370228239102289, "clip_ratio/high_mean": 0.004810697151697241, "clip_ratio/low_mean": 0.0015977961429598508, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006408493412891403, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 917.0, "completions/mean_length": 705.2250366210938, "completions/mean_terminated_length": 513.9599609375, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.09458164777606726, "epoch": 0.6422070534698521, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.025813229382038116, "kl": 0.01941605663159862, "learning_rate": 3.2252559726962456e-06, "loss": 0.0165, "num_tokens": 119295829.0, "reward": 1.225000023841858, "reward_std": 0.9736924767494202, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.997593402862549, "sampling/importance_sampling_ratio/mean": 0.5988240242004395, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.178332805633545, "sampling/sampling_logp_difference/mean": 0.01354488730430603, "step": 1129, "step_time": 72.35544136585668 }, { "clip_ratio/high_max": 0.010410382295958698, "clip_ratio/high_mean": 0.002485140794306062, "clip_ratio/low_mean": 0.0015625000323780114, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00404764074482955, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 961.1749877929688, "completions/mean_terminated_length": 709.875, "completions/min_length": 397.0, "completions/min_terminated_length": 397.0, "entropy": 0.0740975053049624, "epoch": 0.6427758816837316, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0304284505546093, "kl": 0.01011479098815471, "learning_rate": 3.220136518771331e-06, "loss": -0.0508, "num_tokens": 119400276.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.719658136367798, "sampling/importance_sampling_ratio/mean": 0.48354750871658325, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4519598484039307, "sampling/sampling_logp_difference/mean": 0.01049049198627472, "step": 1130, "step_time": 73.80978149431758 }, { "clip_ratio/high_max": 0.0162881818250753, "clip_ratio/high_mean": 0.006693736897432245, "clip_ratio/low_mean": 0.002587890630820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009281627542804927, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 932.0, "completions/mean_length": 815.1749877929688, "completions/mean_terminated_length": 584.368408203125, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.13481975346803665, "epoch": 0.643344709897611, "frac_reward_zero_std": 0.0, "grad_norm": 0.02760753035545349, "kl": 0.01475902006495744, "learning_rate": 3.2150170648464165e-06, "loss": 0.0608, "num_tokens": 119510411.0, "reward": 0.925000011920929, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9767937660217285, "sampling/importance_sampling_ratio/mean": 0.3761041760444641, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9628863334655762, "sampling/sampling_logp_difference/mean": 0.015352611429989338, "step": 1131, "step_time": 74.10151539091021 }, { "clip_ratio/high_max": 0.01555672864196822, "clip_ratio/high_mean": 0.004015731246909127, "clip_ratio/low_mean": 0.0014160156424622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005431746976682916, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 811.0, "completions/mean_length": 781.6749877929688, "completions/mean_terminated_length": 513.8421020507812, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.09137316979467869, "epoch": 0.6439135381114903, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02014363370835781, "kl": 0.015718435868620872, "learning_rate": 3.2098976109215015e-06, "loss": 0.0539, "num_tokens": 119614494.0, "reward": 0.875, "reward_std": 0.9657600522041321, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.471722364425659, "sampling/importance_sampling_ratio/mean": 0.4441785216331482, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.380983829498291, "sampling/sampling_logp_difference/mean": 0.012273021042346954, "step": 1132, "step_time": 75.81966279889457 }, { "clip_ratio/high_max": 0.012091246957425028, "clip_ratio/high_mean": 0.004540983645711094, "clip_ratio/low_mean": 0.0021242721923044883, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006665255787083879, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 938.0, "completions/mean_length": 790.5, "completions/mean_terminated_length": 579.2380981445312, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.09056946635246277, "epoch": 0.6444823663253697, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03346464782953262, "kl": 0.0237855986924842, "learning_rate": 3.204778156996587e-06, "loss": -0.1438, "num_tokens": 119719154.0, "reward": 0.9750000238418579, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.6633715629577637, "sampling/importance_sampling_ratio/mean": 0.48226016759872437, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2316603660583496, "sampling/sampling_logp_difference/mean": 0.013699740171432495, "step": 1133, "step_time": 71.0017053373158 }, { "clip_ratio/high_max": 0.017398100695572793, "clip_ratio/high_mean": 0.006519337999634445, "clip_ratio/low_mean": 0.0019465281293378212, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008465865976177156, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 708.7999877929688, "completions/mean_terminated_length": 557.0370483398438, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.1441155057400465, "epoch": 0.6450511945392492, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021416278555989265, "kl": 0.014418632024899125, "learning_rate": 3.1996587030716724e-06, "loss": -0.0107, "num_tokens": 119805394.0, "reward": 1.274999976158142, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 1.1578400135040283, "sampling/importance_sampling_ratio/mean": 0.28790608048439026, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.046597480773926, "sampling/sampling_logp_difference/mean": 0.017386922612786293, "step": 1134, "step_time": 70.89740920183249 }, { "clip_ratio/high_max": 0.012854757835157216, "clip_ratio/high_mean": 0.00413767337158788, "clip_ratio/low_mean": 0.0020299019233789295, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006167575251311064, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 840.6000366210938, "completions/mean_terminated_length": 565.5, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.07830260321497917, "epoch": 0.6456200227531286, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013406009413301945, "kl": 0.010206229228060693, "learning_rate": 3.194539249146758e-06, "loss": 0.0852, "num_tokens": 119910410.0, "reward": 0.7250000238418579, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.1784034967422485, "sampling/importance_sampling_ratio/mean": 0.3149968087673187, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7539329528808594, "sampling/sampling_logp_difference/mean": 0.011337150819599628, "step": 1135, "step_time": 74.51565821096301 }, { "clip_ratio/high_max": 0.010723798419348896, "clip_ratio/high_mean": 0.00531353399856016, "clip_ratio/low_mean": 0.001847370840550866, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.007160904991906136, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 851.9500122070312, "completions/mean_terminated_length": 737.25, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.09350576624274254, "epoch": 0.646188850967008, "frac_reward_zero_std": 0.0, "grad_norm": 0.029092108830809593, "kl": 0.01178603817243129, "learning_rate": 3.189419795221843e-06, "loss": 0.0469, "num_tokens": 120008512.0, "reward": 1.125, "reward_std": 0.9657599329948425, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.8673946857452393, "sampling/importance_sampling_ratio/mean": 0.3492441773414612, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.02264404296875, "sampling/sampling_logp_difference/mean": 0.012850627303123474, "step": 1136, "step_time": 75.21058978303336 }, { "clip_ratio/high_max": 0.012445723405107856, "clip_ratio/high_mean": 0.0037237240903778, "clip_ratio/low_mean": 0.001660156252910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005383880285080522, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 926.75, "completions/mean_terminated_length": 746.1428833007812, "completions/min_length": 485.0, "completions/min_terminated_length": 485.0, "entropy": 0.07803539047017694, "epoch": 0.6467576791808873, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026086224243044853, "kl": 0.013735832180827856, "learning_rate": 3.1843003412969287e-06, "loss": 0.085, "num_tokens": 120119590.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.3479607105255127, "sampling/importance_sampling_ratio/mean": 0.4704141616821289, "sampling/importance_sampling_ratio/min": 0.0003599400515668094, "sampling/sampling_logp_difference/max": 2.972367525100708, "sampling/sampling_logp_difference/mean": 0.010845289565622807, "step": 1137, "step_time": 73.49049440701492 }, { "clip_ratio/high_max": 0.0018899617716670036, "clip_ratio/high_mean": 0.00037799235724378377, "clip_ratio/low_mean": 0.0011962890712311491, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001574281428474933, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 923.0499877929688, "completions/mean_terminated_length": 620.2000122070312, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.06025538081303239, "epoch": 0.6473265073947668, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.010184752754867077, "kl": 0.007495883502997458, "learning_rate": 3.1791808873720137e-06, "loss": 0.0838, "num_tokens": 120231184.0, "reward": 0.4749999940395355, "reward_std": 0.8469401597976685, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.6257455348968506, "sampling/importance_sampling_ratio/mean": 0.39763563871383667, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.668412446975708, "sampling/sampling_logp_difference/mean": 0.009682046249508858, "step": 1138, "step_time": 74.14570841402747 }, { "clip_ratio/high_max": 0.021016768761910498, "clip_ratio/high_mean": 0.007255001604789868, "clip_ratio/low_mean": 0.0023925781424622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009647579747252166, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 862.6000366210938, "completions/mean_terminated_length": 644.2352905273438, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.1379879703745246, "epoch": 0.6478953356086462, "frac_reward_zero_std": 0.0, "grad_norm": 0.017172468826174736, "kl": 0.0134090791689232, "learning_rate": 3.174061433447099e-06, "loss": 0.015, "num_tokens": 120342336.0, "reward": 0.7250000238418579, "reward_std": 0.905467689037323, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.7935283184051514, "sampling/importance_sampling_ratio/mean": 0.31326597929000854, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8761253356933594, "sampling/sampling_logp_difference/mean": 0.015551254153251648, "step": 1139, "step_time": 78.00251052295789 }, { "clip_ratio/high_max": 0.01886308111716062, "clip_ratio/high_mean": 0.007465052360203117, "clip_ratio/low_mean": 0.0026138775574509054, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01007892971392721, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 720.1749877929688, "completions/mean_terminated_length": 573.888916015625, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.1456641424447298, "epoch": 0.6484641638225256, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.028899114578962326, "kl": 0.013633465860038996, "learning_rate": 3.168941979522184e-06, "loss": -0.1084, "num_tokens": 120451951.0, "reward": 1.125, "reward_std": 0.8825225830078125, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.5624563694000244, "sampling/importance_sampling_ratio/mean": 0.3480316698551178, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.118725299835205, "sampling/sampling_logp_difference/mean": 0.01767158694565296, "step": 1140, "step_time": 77.98597899801098 }, { "clip_ratio/high_max": 0.011465038929600269, "clip_ratio/high_mean": 0.002987539366586134, "clip_ratio/low_mean": 0.0015869140588620212, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004574453458189964, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 734.2000122070312, "completions/mean_terminated_length": 610.0, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "entropy": 0.11357378773391247, "epoch": 0.649032992036405, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.023978281766176224, "kl": 0.011813800781965256, "learning_rate": 3.16382252559727e-06, "loss": -0.0636, "num_tokens": 120556159.0, "reward": 1.375, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.8131471872329712, "sampling/importance_sampling_ratio/mean": 0.2994818687438965, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5462210178375244, "sampling/sampling_logp_difference/mean": 0.014420566149055958, "step": 1141, "step_time": 72.75827878690325 }, { "clip_ratio/high_max": 0.008433594834059477, "clip_ratio/high_mean": 0.0016867189842741936, "clip_ratio/low_mean": 0.0014404296671273187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003127148636849597, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 870.4249877929688, "completions/mean_terminated_length": 551.4615478515625, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.08785956725478172, "epoch": 0.6496018202502845, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.015045818872749805, "kl": 0.012214013608172536, "learning_rate": 3.158703071672355e-06, "loss": 0.0027, "num_tokens": 120659128.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 1.2678554058074951, "sampling/importance_sampling_ratio/mean": 0.36685457825660706, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8868882656097412, "sampling/sampling_logp_difference/mean": 0.012066087685525417, "step": 1142, "step_time": 73.25982502498664 }, { "clip_ratio/high_max": 0.011716973735019565, "clip_ratio/high_mean": 0.0030090423970250413, "clip_ratio/low_mean": 0.0024902344011934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005499276769114658, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 909.0, "completions/mean_length": 751.1000366210938, "completions/mean_terminated_length": 587.3599853515625, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.13343470357358456, "epoch": 0.6501706484641638, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03916472941637039, "kl": 0.015298678074032068, "learning_rate": 3.1535836177474404e-06, "loss": -0.0664, "num_tokens": 120765508.0, "reward": 1.25, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.6331756114959717, "sampling/importance_sampling_ratio/mean": 0.4599221348762512, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8666218519210815, "sampling/sampling_logp_difference/mean": 0.0168264489620924, "step": 1143, "step_time": 74.22955738310702 }, { "clip_ratio/high_max": 0.01743489393265918, "clip_ratio/high_mean": 0.006311304372502491, "clip_ratio/low_mean": 0.0012207031468278728, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007532007584813982, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 692.9000244140625, "completions/mean_terminated_length": 533.4815063476562, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.09090954251587391, "epoch": 0.6507394766780432, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026396840810775757, "kl": 0.015133413835428655, "learning_rate": 3.1484641638225255e-06, "loss": -0.0219, "num_tokens": 120871704.0, "reward": 0.9750000238418579, "reward_std": 0.8002403378486633, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.665386438369751, "sampling/importance_sampling_ratio/mean": 0.4121052622795105, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.743715763092041, "sampling/sampling_logp_difference/mean": 0.013697969727218151, "step": 1144, "step_time": 73.93270216067322 }, { "clip_ratio/high_max": 0.01609527296386659, "clip_ratio/high_mean": 0.005759505118476227, "clip_ratio/low_mean": 0.0015380859331344254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007297590986127034, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 969.0, "completions/mean_length": 821.9249877929688, "completions/mean_terminated_length": 574.9444580078125, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.12379042897373438, "epoch": 0.6513083048919226, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024885904043912888, "kl": 0.015399130061268806, "learning_rate": 3.1433447098976113e-06, "loss": 0.0479, "num_tokens": 120985981.0, "reward": 0.875, "reward_std": 0.9919548630714417, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.868312120437622, "sampling/importance_sampling_ratio/mean": 0.449162095785141, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9337186813354492, "sampling/sampling_logp_difference/mean": 0.015355387702584267, "step": 1145, "step_time": 80.75337002798915 }, { "clip_ratio/high_max": 0.012559565482661128, "clip_ratio/high_mean": 0.004540080291917548, "clip_ratio/low_mean": 0.001025390658469405, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00556547095766291, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 884.0, "completions/mean_length": 784.6000366210938, "completions/mean_terminated_length": 588.727294921875, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.0611472581513226, "epoch": 0.6518771331058021, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021837608888745308, "kl": 0.008391421579290181, "learning_rate": 3.1382252559726963e-06, "loss": 0.016, "num_tokens": 121082477.0, "reward": 1.100000023841858, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.5812489986419678, "sampling/importance_sampling_ratio/mean": 0.6042698621749878, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8761181831359863, "sampling/sampling_logp_difference/mean": 0.009020009078085423, "step": 1146, "step_time": 71.39691430702806 }, { "clip_ratio/high_max": 0.009168239426799119, "clip_ratio/high_mean": 0.0020412890589796007, "clip_ratio/low_mean": 0.0018310546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038723437464796007, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 855.0499877929688, "completions/mean_terminated_length": 716.8181762695312, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.1502849366515875, "epoch": 0.6524459613196815, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.013096318580210209, "kl": 0.011805444024503231, "learning_rate": 3.1331058020477818e-06, "loss": -0.0489, "num_tokens": 121192631.0, "reward": 1.0750000476837158, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057363510131836, "sampling/importance_sampling_ratio/max": 1.8221684694290161, "sampling/importance_sampling_ratio/mean": 0.424126535654068, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6519443988800049, "sampling/sampling_logp_difference/mean": 0.01675618253648281, "step": 1147, "step_time": 75.2786908110138 }, { "clip_ratio/high_max": 0.009385784738697112, "clip_ratio/high_mean": 0.0030605562642449513, "clip_ratio/low_mean": 0.0018066406264551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0048671968397684395, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 902.0, "completions/mean_length": 774.1749877929688, "completions/mean_terminated_length": 498.0526428222656, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.08685014490038157, "epoch": 0.6530147895335608, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014683344401419163, "kl": 0.008987665583845228, "learning_rate": 3.1279863481228668e-06, "loss": 0.0598, "num_tokens": 121294158.0, "reward": 0.925000011920929, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.9841723442077637, "sampling/importance_sampling_ratio/mean": 0.4154167175292969, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6528596878051758, "sampling/sampling_logp_difference/mean": 0.011246412992477417, "step": 1148, "step_time": 72.04374071373604 }, { "clip_ratio/high_max": 0.018732487224042416, "clip_ratio/high_mean": 0.006427687301766127, "clip_ratio/low_mean": 0.0014648437645519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007892530877143145, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 793.0499877929688, "completions/mean_terminated_length": 562.1000366210938, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.11540685500949621, "epoch": 0.6535836177474402, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02230204828083515, "kl": 0.011793486657552421, "learning_rate": 3.1228668941979526e-06, "loss": 0.0631, "num_tokens": 121401152.0, "reward": 0.824999988079071, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.815995216369629, "sampling/importance_sampling_ratio/mean": 0.4523610472679138, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.911067247390747, "sampling/sampling_logp_difference/mean": 0.01514976192265749, "step": 1149, "step_time": 74.71203151182272 }, { "clip_ratio/high_max": 0.004475450841709971, "clip_ratio/high_mean": 0.0008950901683419943, "clip_ratio/low_mean": 0.001440429690774181, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002335519850021228, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 897.0, "completions/mean_length": 778.0, "completions/mean_terminated_length": 506.1052551269531, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.11785013508051634, "epoch": 0.6541524459613197, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.0075538079254329205, "kl": 0.0129592195735313, "learning_rate": 3.1177474402730376e-06, "loss": 0.041, "num_tokens": 121508576.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.4226324558258057, "sampling/importance_sampling_ratio/mean": 0.3912096619606018, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1977181434631348, "sampling/sampling_logp_difference/mean": 0.01476664561778307, "step": 1150, "step_time": 74.18815149692819 }, { "clip_ratio/high_max": 0.008494057663483545, "clip_ratio/high_mean": 0.0026822802756214514, "clip_ratio/low_mean": 0.0018554687849245965, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004537748987786472, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 918.6000366210938, "completions/mean_terminated_length": 640.727294921875, "completions/min_length": 392.0, "completions/min_terminated_length": 392.0, "entropy": 0.062098993454128504, "epoch": 0.6547212741751991, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02137557789683342, "kl": 0.0183422050322406, "learning_rate": 3.112627986348123e-06, "loss": 0.0785, "num_tokens": 121618256.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.313446044921875, "sampling/importance_sampling_ratio/mean": 0.5604162216186523, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3372437953948975, "sampling/sampling_logp_difference/mean": 0.008953506126999855, "step": 1151, "step_time": 73.17456792714074 }, { "clip_ratio/high_max": 0.016074434388428926, "clip_ratio/high_mean": 0.005406754615250975, "clip_ratio/low_mean": 0.0005371093757275958, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005943864060100168, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 754.2250366210938, "completions/mean_terminated_length": 592.3599853515625, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.1133362827822566, "epoch": 0.6552901023890785, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01399705745279789, "kl": 0.01143324445001781, "learning_rate": 3.107508532423208e-06, "loss": -0.0453, "num_tokens": 121727249.0, "reward": 1.149999976158142, "reward_std": 0.9486833214759827, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.4727035760879517, "sampling/importance_sampling_ratio/mean": 0.3420100808143616, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6750948429107666, "sampling/sampling_logp_difference/mean": 0.01316137332469225, "step": 1152, "step_time": 78.17896283790469 }, { "clip_ratio/high_max": 0.011465898365713656, "clip_ratio/high_mean": 0.004765803838381544, "clip_ratio/low_mean": 0.001405420342052821, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006171224114950746, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 825.0, "completions/mean_length": 778.0, "completions/mean_terminated_length": 555.4285888671875, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 0.08179185725748539, "epoch": 0.6558589306029579, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03080887161195278, "kl": 0.017795899650081992, "learning_rate": 3.102389078498294e-06, "loss": 0.0231, "num_tokens": 121827497.0, "reward": 1.0, "reward_std": 0.9870962500572205, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.6909308433532715, "sampling/importance_sampling_ratio/mean": 0.4476291239261627, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4218478202819824, "sampling/sampling_logp_difference/mean": 0.011332893744111061, "step": 1153, "step_time": 73.31016257498413 }, { "clip_ratio/high_max": 0.012236874375957996, "clip_ratio/high_mean": 0.004780742659932002, "clip_ratio/low_mean": 0.0016845703230501385, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006465312966611236, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 974.0, "completions/mean_length": 789.0250244140625, "completions/mean_terminated_length": 615.3478393554688, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.12385796662420034, "epoch": 0.6564277588168373, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021454811096191406, "kl": 0.01493109145667404, "learning_rate": 3.097269624573379e-06, "loss": 0.0583, "num_tokens": 121936706.0, "reward": 1.125, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.5304629802703857, "sampling/importance_sampling_ratio/mean": 0.3769783079624176, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.116804838180542, "sampling/sampling_logp_difference/mean": 0.014812964014708996, "step": 1154, "step_time": 73.4814707946498 }, { "clip_ratio/high_max": 0.013560993422288448, "clip_ratio/high_mean": 0.00383623072411865, "clip_ratio/low_mean": 0.0014160156133584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005252246337477118, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 918.0, "completions/mean_length": 922.0250244140625, "completions/mean_terminated_length": 653.1818237304688, "completions/min_length": 496.0, "completions/min_terminated_length": 496.0, "entropy": 0.14471673406660557, "epoch": 0.6569965870307167, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.012257913127541542, "kl": 0.011610995279625058, "learning_rate": 3.0921501706484644e-06, "loss": 0.0224, "num_tokens": 122051363.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.7444870471954346, "sampling/importance_sampling_ratio/mean": 0.34451255202293396, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8912222385406494, "sampling/sampling_logp_difference/mean": 0.015890253707766533, "step": 1155, "step_time": 76.39664198155515 }, { "clip_ratio/high_max": 0.01372595236171037, "clip_ratio/high_mean": 0.0030513482925016433, "clip_ratio/low_mean": 0.0009765625018189894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004027910836157389, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 834.25, "completions/mean_terminated_length": 577.5294189453125, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "entropy": 0.12030930910259485, "epoch": 0.6575654152445961, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.007117409259080887, "kl": 0.012193244532682002, "learning_rate": 3.0870307167235494e-06, "loss": 0.004, "num_tokens": 122161469.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.9106359481811523, "sampling/importance_sampling_ratio/mean": 0.5363341569900513, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4620704650878906, "sampling/sampling_logp_difference/mean": 0.01455090381205082, "step": 1156, "step_time": 76.10014591482468 }, { "clip_ratio/high_max": 0.013119019044097513, "clip_ratio/high_mean": 0.004422098281793296, "clip_ratio/low_mean": 0.0037237900542095304, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008145888336002827, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 960.0, "completions/mean_length": 829.6500244140625, "completions/mean_terminated_length": 635.2999877929688, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "entropy": 0.15092395059764385, "epoch": 0.6581342434584755, "frac_reward_zero_std": 0.0, "grad_norm": 0.04066362977027893, "kl": 0.013554878765717149, "learning_rate": 3.0819112627986353e-06, "loss": 0.0056, "num_tokens": 122275487.0, "reward": 0.800000011920929, "reward_std": 0.8828856945037842, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.7493882179260254, "sampling/importance_sampling_ratio/mean": 0.37444716691970825, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9644546508789062, "sampling/sampling_logp_difference/mean": 0.016355810686945915, "step": 1157, "step_time": 75.39119090186432 }, { "clip_ratio/high_max": 0.01826726016588509, "clip_ratio/high_mean": 0.006132495269412175, "clip_ratio/low_mean": 0.001641945595110883, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007774440920911729, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 813.5, "completions/mean_terminated_length": 603.0, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.09240898489952087, "epoch": 0.658703071672355, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03423900157213211, "kl": 0.015217104810290039, "learning_rate": 3.0767918088737203e-06, "loss": -0.003, "num_tokens": 122386939.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.192408800125122, "sampling/importance_sampling_ratio/mean": 0.43992432951927185, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5341408252716064, "sampling/sampling_logp_difference/mean": 0.013388982973992825, "step": 1158, "step_time": 74.5405819141306 }, { "clip_ratio/high_max": 0.01600740768481046, "clip_ratio/high_mean": 0.006194444169523194, "clip_ratio/low_mean": 0.0019299232590128668, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008124367508571595, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 747.5250244140625, "completions/mean_terminated_length": 563.2083740234375, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 0.07170442957431078, "epoch": 0.6592718998862344, "frac_reward_zero_std": 0.0, "grad_norm": 0.03567669913172722, "kl": 0.011808257200755179, "learning_rate": 3.0716723549488057e-06, "loss": 0.0976, "num_tokens": 122478560.0, "reward": 1.125, "reward_std": 0.9657599329948425, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.9773058891296387, "sampling/importance_sampling_ratio/mean": 0.4859848916530609, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1183853149414062, "sampling/sampling_logp_difference/mean": 0.011188723146915436, "step": 1159, "step_time": 69.90982317691669 }, { "clip_ratio/high_max": 0.013204708404373378, "clip_ratio/high_mean": 0.0034893892152467743, "clip_ratio/low_mean": 0.0021240234636934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005613412708044052, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 821.6000366210938, "completions/mean_terminated_length": 619.2000122070312, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.09553876984864473, "epoch": 0.6598407281001137, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024531211704015732, "kl": 0.01209594588726759, "learning_rate": 3.0665529010238907e-06, "loss": 0.0504, "num_tokens": 122587192.0, "reward": 0.9000000357627869, "reward_std": 0.9554163813591003, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.9321954250335693, "sampling/importance_sampling_ratio/mean": 0.414519727230072, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.734462857246399, "sampling/sampling_logp_difference/mean": 0.013121956959366798, "step": 1160, "step_time": 73.6783834816888 }, { "clip_ratio/high_max": 0.009093045606277883, "clip_ratio/high_mean": 0.0018924643663922325, "clip_ratio/low_mean": 0.0021410768677014858, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0040335412486456335, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 776.0, "completions/mean_length": 829.9500122070312, "completions/mean_terminated_length": 567.4117431640625, "completions/min_length": 444.0, "completions/min_terminated_length": 444.0, "entropy": 0.09710322506725788, "epoch": 0.6604095563139932, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01357266865670681, "kl": 0.010478323267307132, "learning_rate": 3.0614334470989766e-06, "loss": -0.0386, "num_tokens": 122693022.0, "reward": 0.824999988079071, "reward_std": 0.9841695427894592, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.6958245038986206, "sampling/importance_sampling_ratio/mean": 0.3253259062767029, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6875364780426025, "sampling/sampling_logp_difference/mean": 0.013091444969177246, "step": 1161, "step_time": 75.65381831559353 }, { "clip_ratio/high_max": 0.013495312072336674, "clip_ratio/high_mean": 0.005904543271753937, "clip_ratio/low_mean": 0.0028076171875, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.008712160575669259, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 899.0, "completions/mean_length": 848.3250122070312, "completions/mean_terminated_length": 654.1578979492188, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.1103971553966403, "epoch": 0.6609783845278726, "frac_reward_zero_std": 0.0, "grad_norm": 0.0237701628357172, "kl": 0.012739910569507629, "learning_rate": 3.0563139931740616e-06, "loss": -0.0553, "num_tokens": 122804355.0, "reward": 0.9000000357627869, "reward_std": 0.9818872809410095, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.9766672849655151, "sampling/importance_sampling_ratio/mean": 0.3505663275718689, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0740039348602295, "sampling/sampling_logp_difference/mean": 0.014539489522576332, "step": 1162, "step_time": 74.73498749057762 }, { "clip_ratio/high_max": 0.010115407465491444, "clip_ratio/high_mean": 0.004813067862414755, "clip_ratio/low_mean": 0.002197265646827873, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.007010333298239857, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 953.0, "completions/mean_length": 848.4750366210938, "completions/mean_terminated_length": 611.0, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.12428695615381002, "epoch": 0.661547212741752, "frac_reward_zero_std": 0.0, "grad_norm": 0.04475189372897148, "kl": 0.01616037351777777, "learning_rate": 3.0511945392491466e-06, "loss": -0.0915, "num_tokens": 122908278.0, "reward": 0.625, "reward_std": 0.8066231608390808, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.736426591873169, "sampling/importance_sampling_ratio/mean": 0.52602219581604, "sampling/importance_sampling_ratio/min": 0.0038035977631807327, "sampling/sampling_logp_difference/max": 2.6370506286621094, "sampling/sampling_logp_difference/mean": 0.013951546512544155, "step": 1163, "step_time": 74.03077623806894 }, { "clip_ratio/high_max": 0.012232558918185532, "clip_ratio/high_mean": 0.00468281251960434, "clip_ratio/low_mean": 0.0017105104197980836, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0063933229830581695, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 734.2000122070312, "completions/mean_terminated_length": 578.1538696289062, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.13080775178968906, "epoch": 0.6621160409556314, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026191972196102142, "kl": 0.012490244349464774, "learning_rate": 3.046075085324232e-06, "loss": -0.0006, "num_tokens": 123010774.0, "reward": 1.024999976158142, "reward_std": 0.8619447350502014, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.5528035163879395, "sampling/importance_sampling_ratio/mean": 0.3407014012336731, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.21648907661438, "sampling/sampling_logp_difference/mean": 0.01539869885891676, "step": 1164, "step_time": 74.74208681983873 }, { "clip_ratio/high_max": 0.012541255098767579, "clip_ratio/high_mean": 0.004078874888364226, "clip_ratio/low_mean": 0.000732421893189894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004811296792468056, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 902.7000122070312, "completions/mean_terminated_length": 700.5333862304688, "completions/min_length": 386.0, "completions/min_terminated_length": 386.0, "entropy": 0.09955190028995275, "epoch": 0.6626848691695109, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.020898116752505302, "kl": 0.009727975935675204, "learning_rate": 3.0409556313993175e-06, "loss": 0.0321, "num_tokens": 123120386.0, "reward": 0.75, "reward_std": 0.9805806279182434, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.927699327468872, "sampling/importance_sampling_ratio/mean": 0.5711637735366821, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3350744247436523, "sampling/sampling_logp_difference/mean": 0.01128208078444004, "step": 1165, "step_time": 73.6171694220975 }, { "clip_ratio/high_max": 0.012927899369969964, "clip_ratio/high_mean": 0.004396405041916296, "clip_ratio/low_mean": 0.001391601559589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005788006557850167, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 634.1500244140625, "completions/mean_terminated_length": 520.9677124023438, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.11418679729104042, "epoch": 0.6632536973833902, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02014118619263172, "kl": 0.012439627258572727, "learning_rate": 3.035836177474403e-06, "loss": -0.0534, "num_tokens": 123221936.0, "reward": 1.4250000715255737, "reward_std": 0.8439072966575623, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.7611000537872314, "sampling/importance_sampling_ratio/mean": 0.495109885931015, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6586462259292603, "sampling/sampling_logp_difference/mean": 0.015157905407249928, "step": 1166, "step_time": 69.16691639763303 }, { "clip_ratio/high_max": 0.010345803864765912, "clip_ratio/high_mean": 0.0025602492096368223, "clip_ratio/low_mean": 0.0009033203314174898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034635694755706936, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 813.4750366210938, "completions/mean_terminated_length": 556.1666870117188, "completions/min_length": 419.0, "completions/min_terminated_length": 419.0, "entropy": 0.07198436744511127, "epoch": 0.6638225255972696, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01714342087507248, "kl": 0.00884473737096414, "learning_rate": 3.030716723549488e-06, "loss": 0.0196, "num_tokens": 123319619.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.271829128265381, "sampling/importance_sampling_ratio/mean": 0.46453413367271423, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9860222339630127, "sampling/sampling_logp_difference/mean": 0.010007950477302074, "step": 1167, "step_time": 70.61877636378631 }, { "clip_ratio/high_max": 0.017049489310011268, "clip_ratio/high_mean": 0.0061152546550147235, "clip_ratio/low_mean": 0.003013565205037594, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009128819452598691, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.0, "completions/mean_length": 805.2000122070312, "completions/mean_terminated_length": 626.1818237304688, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.13744588196277618, "epoch": 0.664391353811149, "frac_reward_zero_std": 0.0, "grad_norm": 0.037577513605356216, "kl": 0.011040647223126143, "learning_rate": 3.0255972696245733e-06, "loss": 0.1522, "num_tokens": 123416739.0, "reward": 0.8500000238418579, "reward_std": 0.8638019561767578, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.92732572555542, "sampling/importance_sampling_ratio/mean": 0.5151960253715515, "sampling/importance_sampling_ratio/min": 0.00019774373504333198, "sampling/sampling_logp_difference/max": 2.9508888721466064, "sampling/sampling_logp_difference/mean": 0.015307160094380379, "step": 1168, "step_time": 72.29866404994391 }, { "clip_ratio/high_max": 0.01245894463500008, "clip_ratio/high_mean": 0.003371996179339476, "clip_ratio/low_mean": 0.0007568359578726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004128832195419818, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 908.0, "completions/mean_length": 856.3250122070312, "completions/mean_terminated_length": 508.0769348144531, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.08220859244465828, "epoch": 0.6649601820250285, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014265299774706364, "kl": 0.010730427806265652, "learning_rate": 3.0204778156996588e-06, "loss": 0.0387, "num_tokens": 123526784.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.7092647552490234, "sampling/importance_sampling_ratio/mean": 0.24276158213615417, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.096158504486084, "sampling/sampling_logp_difference/mean": 0.011162654496729374, "step": 1169, "step_time": 77.2471591483336 }, { "clip_ratio/high_max": 0.010796405840665102, "clip_ratio/high_mean": 0.003670369944302365, "clip_ratio/low_mean": 0.003118479988188483, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.006788850121665746, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 847.5250244140625, "completions/mean_terminated_length": 671.0499877929688, "completions/min_length": 427.0, "completions/min_terminated_length": 427.0, "entropy": 0.14589203521609306, "epoch": 0.6655290102389079, "frac_reward_zero_std": 0.0, "grad_norm": 0.027599871158599854, "kl": 0.014119971543550491, "learning_rate": 3.015358361774744e-06, "loss": -0.0093, "num_tokens": 123629317.0, "reward": 0.7750000357627869, "reward_std": 0.8619447350502014, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.8205697536468506, "sampling/importance_sampling_ratio/mean": 0.35428953170776367, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.377474546432495, "sampling/sampling_logp_difference/mean": 0.015379360876977444, "step": 1170, "step_time": 74.5722189838998 }, { "clip_ratio/high_max": 0.011418980429880321, "clip_ratio/high_mean": 0.0027468005719128996, "clip_ratio/low_mean": 0.002050781258731149, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004797581786988303, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 709.625, "completions/mean_terminated_length": 521.0, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.1166713023558259, "epoch": 0.6660978384527872, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015317687764763832, "kl": 0.013126256060786545, "learning_rate": 3.010238907849829e-06, "loss": -0.02, "num_tokens": 123732198.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.8079445362091064, "sampling/importance_sampling_ratio/mean": 0.28792548179626465, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.203024387359619, "sampling/sampling_logp_difference/mean": 0.01572001911699772, "step": 1171, "step_time": 72.12094357679598 }, { "clip_ratio/high_max": 0.014976544392993674, "clip_ratio/high_mean": 0.006756963164662011, "clip_ratio/low_mean": 0.001436526497855084, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008193489571567625, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 616.0, "completions/mean_length": 586.7250366210938, "completions/mean_terminated_length": 440.9666748046875, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.110214127227664, "epoch": 0.6666666666666666, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018120678141713142, "kl": 0.01527327299118042, "learning_rate": 3.0051194539249146e-06, "loss": 0.0034, "num_tokens": 123832227.0, "reward": 1.475000023841858, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.7250000238418579, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 0.8998109698295593, "sampling/importance_sampling_ratio/mean": 0.2376450002193451, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.336970090866089, "sampling/sampling_logp_difference/mean": 0.015225984156131744, "step": 1172, "step_time": 71.48474063607864 }, { "clip_ratio/high_max": 0.01429369195830077, "clip_ratio/high_mean": 0.007053504159557633, "clip_ratio/low_mean": 0.001857600022049155, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008911104232538491, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 918.0, "completions/mean_length": 706.9249877929688, "completions/mean_terminated_length": 536.1923217773438, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.12166284676641226, "epoch": 0.6672354948805461, "frac_reward_zero_std": 0.0, "grad_norm": 0.02521562948822975, "kl": 0.017566916532814503, "learning_rate": 3e-06, "loss": -0.0325, "num_tokens": 123940488.0, "reward": 1.25, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.974475860595703, "sampling/importance_sampling_ratio/mean": 0.39544612169265747, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1079111099243164, "sampling/sampling_logp_difference/mean": 0.015137158334255219, "step": 1173, "step_time": 73.11517294123769 }, { "clip_ratio/high_max": 0.017455063934903592, "clip_ratio/high_mean": 0.006609460280742496, "clip_ratio/low_mean": 0.0019336976401973516, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008543157717213035, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 971.0, "completions/mean_length": 762.125, "completions/mean_terminated_length": 636.0370483398438, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.14678263664245605, "epoch": 0.6678043230944255, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02470029518008232, "kl": 0.015325655811466277, "learning_rate": 2.9948805460750855e-06, "loss": -0.0376, "num_tokens": 124047173.0, "reward": 1.2000000476837158, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.9642117023468018, "sampling/importance_sampling_ratio/mean": 0.4511653482913971, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1188552379608154, "sampling/sampling_logp_difference/mean": 0.017149794846773148, "step": 1174, "step_time": 74.20098217879422 }, { "clip_ratio/high_max": 0.008478979725623503, "clip_ratio/high_mean": 0.002332605276023969, "clip_ratio/low_mean": 0.0009277343961002771, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003260339639382437, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 960.0, "completions/mean_length": 809.5750122070312, "completions/mean_terminated_length": 595.1500244140625, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.0723894783295691, "epoch": 0.6683731513083049, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.012909467332065105, "kl": 0.009796464000828564, "learning_rate": 2.9897610921501705e-06, "loss": 0.054, "num_tokens": 124147164.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.3021979331970215, "sampling/importance_sampling_ratio/mean": 0.3323870301246643, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8910255432128906, "sampling/sampling_logp_difference/mean": 0.00999975111335516, "step": 1175, "step_time": 71.53244489175268 }, { "clip_ratio/high_max": 0.016423166496679187, "clip_ratio/high_mean": 0.004239673318807036, "clip_ratio/low_mean": 0.00292830154648982, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007167974894400686, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 791.7750244140625, "completions/mean_terminated_length": 559.5499877929688, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.15381196700036526, "epoch": 0.6689419795221843, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024459905922412872, "kl": 0.018779575591906905, "learning_rate": 2.984641638225256e-06, "loss": 0.0387, "num_tokens": 124261475.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.0187432765960693, "sampling/importance_sampling_ratio/mean": 0.41891422867774963, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9599072933197021, "sampling/sampling_logp_difference/mean": 0.01745593547821045, "step": 1176, "step_time": 76.7185647697188 }, { "clip_ratio/high_max": 0.013292804127559066, "clip_ratio/high_mean": 0.004918373626424, "clip_ratio/low_mean": 0.002172851556679234, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.0070912252413108945, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 942.0, "completions/mean_length": 859.0499877929688, "completions/mean_terminated_length": 657.4444580078125, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.13581401761621237, "epoch": 0.6695108077360638, "frac_reward_zero_std": 0.0, "grad_norm": 0.03818921744823456, "kl": 0.012989540467970073, "learning_rate": 2.9795221843003414e-06, "loss": -0.0232, "num_tokens": 124365733.0, "reward": 0.824999988079071, "reward_std": 0.957761824131012, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.732211947441101, "sampling/importance_sampling_ratio/mean": 0.38314875960350037, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.036116600036621, "sampling/sampling_logp_difference/mean": 0.014201163314282894, "step": 1177, "step_time": 73.55070336000063 }, { "clip_ratio/high_max": 0.013021125923842192, "clip_ratio/high_mean": 0.004773381049744785, "clip_ratio/low_mean": 0.002197265646827873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006970646500121802, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 825.625, "completions/mean_terminated_length": 679.0, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.15660951100289822, "epoch": 0.6700796359499431, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03566037863492966, "kl": 0.01857796951662749, "learning_rate": 2.974402730375427e-06, "loss": -0.0138, "num_tokens": 124481262.0, "reward": 1.0750000476837158, "reward_std": 0.9710555076599121, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.9914910793304443, "sampling/importance_sampling_ratio/mean": 0.4357655942440033, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7121007442474365, "sampling/sampling_logp_difference/mean": 0.017462512478232384, "step": 1178, "step_time": 76.55519841914065 }, { "clip_ratio/high_max": 0.011992369079962373, "clip_ratio/high_mean": 0.0032201822905335575, "clip_ratio/low_mean": 0.0008544921875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0040746744780335575, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 685.375, "completions/mean_terminated_length": 572.5, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.08398031862452626, "epoch": 0.6706484641638225, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.48808908462524414, "kl": 0.07086398382671177, "learning_rate": 2.969283276450512e-06, "loss": 0.1302, "num_tokens": 124580365.0, "reward": 1.399999976158142, "reward_std": 0.8711913824081421, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.462737798690796, "sampling/importance_sampling_ratio/mean": 0.386460542678833, "sampling/importance_sampling_ratio/min": 0.00045030584442429245, "sampling/sampling_logp_difference/max": 5.188479423522949, "sampling/sampling_logp_difference/mean": 0.013295546174049377, "step": 1179, "step_time": 69.63891641423106 }, { "clip_ratio/high_max": 0.014169610163662583, "clip_ratio/high_mean": 0.00610122480429709, "clip_ratio/low_mean": 0.0006347656308207661, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006735990464221686, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 925.0, "completions/mean_length": 750.75, "completions/mean_terminated_length": 682.4375, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.08503913041204214, "epoch": 0.6712172923777019, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020385093986988068, "kl": 0.01341375638730824, "learning_rate": 2.9641638225255973e-06, "loss": 0.0535, "num_tokens": 124673003.0, "reward": 1.25, "reward_std": 0.7762500047683716, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.4078915119171143, "sampling/importance_sampling_ratio/mean": 0.389475017786026, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.425097942352295, "sampling/sampling_logp_difference/mean": 0.012820610776543617, "step": 1180, "step_time": 72.05940336268395 }, { "clip_ratio/high_max": 0.012448520516045392, "clip_ratio/high_mean": 0.003472046897513792, "clip_ratio/low_mean": 0.0019531250145519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005425171897513792, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 843.4000244140625, "completions/mean_terminated_length": 643.7894897460938, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.17036472633481026, "epoch": 0.6717861205915814, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02877630852162838, "kl": 0.014868608093820512, "learning_rate": 2.9590443686006827e-06, "loss": 0.0125, "num_tokens": 124780307.0, "reward": 0.925000011920929, "reward_std": 0.9971111416816711, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9394142627716064, "sampling/importance_sampling_ratio/mean": 0.5335468053817749, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2713170051574707, "sampling/sampling_logp_difference/mean": 0.018456751480698586, "step": 1181, "step_time": 73.48775641387329 }, { "clip_ratio/high_max": 0.004602104105288163, "clip_ratio/high_mean": 0.0009204208254232071, "clip_ratio/low_mean": 0.0002685546896827873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011889755151059944, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 806.0, "completions/mean_length": 749.25, "completions/mean_terminated_length": 500.66668701171875, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.08726496808230877, "epoch": 0.6723549488054608, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.017772769555449486, "kl": 0.014169171277899295, "learning_rate": 2.953924914675768e-06, "loss": -0.065, "num_tokens": 124886877.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.608381509780884, "sampling/importance_sampling_ratio/mean": 0.5029808282852173, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9326092004776, "sampling/sampling_logp_difference/mean": 0.012506640516221523, "step": 1182, "step_time": 72.13400354422629 }, { "clip_ratio/high_max": 0.012652769451960921, "clip_ratio/high_mean": 0.002976253890665248, "clip_ratio/low_mean": 0.0010452892474859254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0040215431654360145, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 945.9249877929688, "completions/mean_terminated_length": 740.0909423828125, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.08255008095875382, "epoch": 0.6729237770193401, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01263026800006628, "kl": 0.008489388506859541, "learning_rate": 2.948805460750853e-06, "loss": 0.0629, "num_tokens": 124998898.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.468752145767212, "sampling/importance_sampling_ratio/mean": 0.4434785842895508, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.462191343307495, "sampling/sampling_logp_difference/mean": 0.010343112051486969, "step": 1183, "step_time": 76.5887440298684 }, { "clip_ratio/high_max": 0.015563580673187971, "clip_ratio/high_mean": 0.0069529401225736365, "clip_ratio/low_mean": 0.0024025393649935722, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009355479385703802, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 869.0, "completions/mean_length": 801.7250366210938, "completions/mean_terminated_length": 619.8636474609375, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.1217383649200201, "epoch": 0.6734926052332195, "frac_reward_zero_std": 0.0, "grad_norm": 0.045802101492881775, "kl": 0.013840315048582852, "learning_rate": 2.943686006825939e-06, "loss": -0.021, "num_tokens": 125101775.0, "reward": 1.0750000476837158, "reward_std": 0.9971111416816711, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057363510131836, "sampling/importance_sampling_ratio/max": 2.59602952003479, "sampling/importance_sampling_ratio/mean": 0.3964647352695465, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9851346015930176, "sampling/sampling_logp_difference/mean": 0.015459171496331692, "step": 1184, "step_time": 75.62340941443108 }, { "clip_ratio/high_max": 0.010620320565067232, "clip_ratio/high_mean": 0.0024537343997508287, "clip_ratio/low_mean": 0.0015299676233553328, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003983702044934034, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 850.0, "completions/mean_length": 722.9500122070312, "completions/mean_terminated_length": 522.25, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.09296337515115738, "epoch": 0.674061433447099, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04385126382112503, "kl": 0.010564008087385446, "learning_rate": 2.938566552901024e-06, "loss": -0.0423, "num_tokens": 125195093.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.124459743499756, "sampling/importance_sampling_ratio/mean": 0.4508700966835022, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.459341049194336, "sampling/sampling_logp_difference/mean": 0.012617652304470539, "step": 1185, "step_time": 68.16365621704608 }, { "clip_ratio/high_max": 0.002871868957299739, "clip_ratio/high_mean": 0.0005743737856391817, "clip_ratio/low_mean": 0.0008544922093278728, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014288659949670546, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 738.2999877929688, "completions/mean_terminated_length": 584.4615478515625, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "entropy": 0.12245634291321039, "epoch": 0.6746302616609784, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.01013592816889286, "kl": 0.012842963333241642, "learning_rate": 2.9334470989761094e-06, "loss": -0.0182, "num_tokens": 125297185.0, "reward": 1.3000000715255737, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.0216856002807617, "sampling/importance_sampling_ratio/mean": 0.2946842312812805, "sampling/importance_sampling_ratio/min": 8.187269122572616e-05, "sampling/sampling_logp_difference/max": 1.7330102920532227, "sampling/sampling_logp_difference/mean": 0.015890732407569885, "step": 1186, "step_time": 69.3433743508067 }, { "clip_ratio/high_max": 0.018263914855197072, "clip_ratio/high_mean": 0.007942346477648243, "clip_ratio/low_mean": 0.0024412060683971504, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010383552638813853, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 824.0, "completions/mean_terminated_length": 704.0, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "entropy": 0.1736643835902214, "epoch": 0.6751990898748578, "frac_reward_zero_std": 0.0, "grad_norm": 0.025205647572875023, "kl": 0.01832470449153334, "learning_rate": 2.9283276450511945e-06, "loss": -0.0439, "num_tokens": 125410361.0, "reward": 1.2000000476837158, "reward_std": 0.9660917520523071, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.2629828453063965, "sampling/importance_sampling_ratio/mean": 0.40387535095214844, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.944890975952148, "sampling/sampling_logp_difference/mean": 0.01808745041489601, "step": 1187, "step_time": 76.84377455129288 }, { "clip_ratio/high_max": 0.0068389056250452995, "clip_ratio/high_mean": 0.001869583735242486, "clip_ratio/low_mean": 0.0038818359607830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00575141966692172, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 972.9000244140625, "completions/mean_terminated_length": 615.2000122070312, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.12961065489798784, "epoch": 0.6757679180887372, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013910630717873573, "kl": 0.019119908043649048, "learning_rate": 2.9232081911262803e-06, "loss": -0.0019, "num_tokens": 125523821.0, "reward": 0.20000000298023224, "reward_std": 0.5638694167137146, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 1.6293158531188965, "sampling/importance_sampling_ratio/mean": 0.22162866592407227, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2309203147888184, "sampling/sampling_logp_difference/mean": 0.013521094806492329, "step": 1188, "step_time": 74.94294265587814 }, { "clip_ratio/high_max": 0.014371208380907774, "clip_ratio/high_mean": 0.004840489069465548, "clip_ratio/low_mean": 0.002606164969620295, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0074466539663262665, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 761.625, "completions/mean_terminated_length": 620.34619140625, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.15359701216220856, "epoch": 0.6763367463026166, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023585470393300056, "kl": 0.013199819601140916, "learning_rate": 2.9180887372013653e-06, "loss": 0.005, "num_tokens": 125620622.0, "reward": 1.25, "reward_std": 0.9540736079216003, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.825563430786133, "sampling/importance_sampling_ratio/mean": 0.45396167039871216, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.63637375831604, "sampling/sampling_logp_difference/mean": 0.01820116490125656, "step": 1189, "step_time": 69.56450769980438 }, { "clip_ratio/high_max": 0.01433429797179997, "clip_ratio/high_mean": 0.005296971183270216, "clip_ratio/low_mean": 0.0016979312495095655, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006994902360020205, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 884.0250244140625, "completions/mean_terminated_length": 712.9444580078125, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "entropy": 0.10761811397969723, "epoch": 0.676905574516496, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02154841646552086, "kl": 0.011118934722617269, "learning_rate": 2.9129692832764508e-06, "loss": -0.1274, "num_tokens": 125721215.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.583493947982788, "sampling/importance_sampling_ratio/mean": 0.448295921087265, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.335526943206787, "sampling/sampling_logp_difference/mean": 0.013187723234295845, "step": 1190, "step_time": 72.23462584731169 }, { "clip_ratio/high_max": 0.014449243433773518, "clip_ratio/high_mean": 0.004038060127641074, "clip_ratio/low_mean": 0.0007568359578726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004794896143721417, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 888.1000366210938, "completions/mean_terminated_length": 661.6000366210938, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.11929427273571491, "epoch": 0.6774744027303754, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025387298315763474, "kl": 0.010332771693356335, "learning_rate": 2.9078498293515358e-06, "loss": 0.1412, "num_tokens": 125825675.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.315507411956787, "sampling/importance_sampling_ratio/mean": 0.5471006631851196, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9822745323181152, "sampling/sampling_logp_difference/mean": 0.013337227515876293, "step": 1191, "step_time": 74.05678616208024 }, { "clip_ratio/high_max": 0.014307478093542159, "clip_ratio/high_mean": 0.004349872237071395, "clip_ratio/low_mean": 0.0034179687354480848, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.007767840870656073, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 967.0, "completions/mean_length": 873.7250366210938, "completions/mean_terminated_length": 561.6154174804688, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.1340171294286847, "epoch": 0.6780432309442548, "frac_reward_zero_std": 0.0, "grad_norm": 0.04455970227718353, "kl": 0.01350794720929116, "learning_rate": 2.9027303754266216e-06, "loss": -0.0882, "num_tokens": 125931416.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.5281622409820557, "sampling/importance_sampling_ratio/mean": 0.3645598888397217, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7365236282348633, "sampling/sampling_logp_difference/mean": 0.015091788955032825, "step": 1192, "step_time": 71.70909988158382 }, { "clip_ratio/high_max": 0.01462525560054928, "clip_ratio/high_mean": 0.004803553965757601, "clip_ratio/low_mean": 0.0027587890799622983, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.007562343147583306, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 874.75, "completions/mean_terminated_length": 564.7692260742188, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.12524456437677145, "epoch": 0.6786120591581343, "frac_reward_zero_std": 0.0, "grad_norm": 0.051593635231256485, "kl": 0.010073733981698751, "learning_rate": 2.8976109215017066e-06, "loss": 0.0325, "num_tokens": 126028718.0, "reward": 0.5250000357627869, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.8260921239852905, "sampling/importance_sampling_ratio/mean": 0.41393333673477173, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7991230487823486, "sampling/sampling_logp_difference/mean": 0.014079106040298939, "step": 1193, "step_time": 70.88176379073411 }, { "clip_ratio/high_max": 0.01512466766871512, "clip_ratio/high_mean": 0.0040278964006574824, "clip_ratio/low_mean": 0.0014892578055878403, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005517154204426333, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 874.2250366210938, "completions/mean_terminated_length": 524.75, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.14789703767746687, "epoch": 0.6791808873720137, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015985239297151566, "kl": 0.01345691119786352, "learning_rate": 2.892491467576792e-06, "loss": 0.0517, "num_tokens": 126134551.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.543900489807129, "sampling/importance_sampling_ratio/mean": 0.3696843683719635, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3165884017944336, "sampling/sampling_logp_difference/mean": 0.014834373258054256, "step": 1194, "step_time": 73.22969893738627 }, { "clip_ratio/high_max": 0.010006849654018879, "clip_ratio/high_mean": 0.002812949300277978, "clip_ratio/low_mean": 0.002026367194048362, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004839316374273039, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 964.1000366210938, "completions/mean_terminated_length": 724.5, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.13178823422640562, "epoch": 0.679749715585893, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014918795786798, "kl": 0.009005074738524854, "learning_rate": 2.887372013651877e-06, "loss": 0.029, "num_tokens": 126246131.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.7849080562591553, "sampling/importance_sampling_ratio/mean": 0.492971807718277, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.353940010070801, "sampling/sampling_logp_difference/mean": 0.013338834047317505, "step": 1195, "step_time": 78.47922349930741 }, { "clip_ratio/high_max": 0.010067722614621744, "clip_ratio/high_mean": 0.0027621471454040147, "clip_ratio/low_mean": 0.0011718750065483619, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00393402211193461, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 783.0, "completions/mean_length": 891.5250244140625, "completions/mean_terminated_length": 582.4166870117188, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "entropy": 0.07255689147859812, "epoch": 0.6803185437997725, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022468704730272293, "kl": 0.008555280801374465, "learning_rate": 2.882252559726963e-06, "loss": 0.075, "num_tokens": 126346760.0, "reward": 0.6000000238418579, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.4997169971466064, "sampling/importance_sampling_ratio/mean": 0.4689866602420807, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8761308193206787, "sampling/sampling_logp_difference/mean": 0.010079827159643173, "step": 1196, "step_time": 72.96110028587282 }, { "clip_ratio/high_max": 0.01304260641336441, "clip_ratio/high_mean": 0.004518336616456509, "clip_ratio/low_mean": 0.0011067354680562858, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005625072168186307, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 804.5499877929688, "completions/mean_terminated_length": 625.0, "completions/min_length": 424.0, "completions/min_terminated_length": 424.0, "entropy": 0.09176288265734911, "epoch": 0.6808873720136519, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02458508498966694, "kl": 0.011928569001611322, "learning_rate": 2.877133105802048e-06, "loss": -0.0101, "num_tokens": 126433630.0, "reward": 1.0, "reward_std": 0.9607689380645752, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.749755620956421, "sampling/importance_sampling_ratio/mean": 0.45961982011795044, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8717933893203735, "sampling/sampling_logp_difference/mean": 0.01177749689668417, "step": 1197, "step_time": 70.623207524186 }, { "clip_ratio/high_max": 0.013965537771582603, "clip_ratio/high_mean": 0.006673797965049744, "clip_ratio/low_mean": 0.002248175052955048, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008921973058022559, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 807.9750366210938, "completions/mean_terminated_length": 678.3599853515625, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "entropy": 0.13472402468323708, "epoch": 0.6814562002275313, "frac_reward_zero_std": 0.0, "grad_norm": 0.01690957136452198, "kl": 0.01739156700205058, "learning_rate": 2.872013651877133e-06, "loss": 0.0238, "num_tokens": 126544557.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.5419793128967285, "sampling/importance_sampling_ratio/mean": 0.3228597640991211, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8187451362609863, "sampling/sampling_logp_difference/mean": 0.016091356053948402, "step": 1198, "step_time": 75.45913472166285 }, { "clip_ratio/high_max": 0.016895027365535498, "clip_ratio/high_mean": 0.005241845297859982, "clip_ratio/low_mean": 0.0008789062558207661, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006120751379057765, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 863.9000244140625, "completions/mean_terminated_length": 647.2941284179688, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.08215861720964313, "epoch": 0.6820250284414107, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.020129084587097168, "kl": 0.010222877433989197, "learning_rate": 2.8668941979522184e-06, "loss": -0.056, "num_tokens": 126650449.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.829664468765259, "sampling/importance_sampling_ratio/mean": 0.5925725698471069, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.132164239883423, "sampling/sampling_logp_difference/mean": 0.010878927074372768, "step": 1199, "step_time": 75.3638927724678 }, { "clip_ratio/high_max": 0.013045794446952641, "clip_ratio/high_mean": 0.004754353081807494, "clip_ratio/low_mean": 0.002514648440410383, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.007269001507665962, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 864.7250366210938, "completions/mean_terminated_length": 670.0555419921875, "completions/min_length": 468.0, "completions/min_terminated_length": 468.0, "entropy": 0.10864113736897707, "epoch": 0.6825938566552902, "frac_reward_zero_std": 0.0, "grad_norm": 0.022053737193346024, "kl": 0.015988483326509595, "learning_rate": 2.8617747440273034e-06, "loss": -0.0053, "num_tokens": 126750534.0, "reward": 0.7750000357627869, "reward_std": 0.9195177555084229, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.4406572580337524, "sampling/importance_sampling_ratio/mean": 0.26583611965179443, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6382322311401367, "sampling/sampling_logp_difference/mean": 0.013534076511859894, "step": 1200, "step_time": 75.95513676246628 }, { "clip_ratio/high_max": 0.0185274031246081, "clip_ratio/high_mean": 0.0063175193790812045, "clip_ratio/low_mean": 0.0022705078044964466, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008588027325458825, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 940.0, "completions/mean_length": 833.875, "completions/mean_terminated_length": 548.6875, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.1839362010359764, "epoch": 0.6831626848691695, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027432646602392197, "kl": 0.01742202928289771, "learning_rate": 2.8566552901023893e-06, "loss": 0.0666, "num_tokens": 126860601.0, "reward": 0.699999988079071, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.593630313873291, "sampling/importance_sampling_ratio/mean": 0.37219589948654175, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7005672454833984, "sampling/sampling_logp_difference/mean": 0.01967880129814148, "step": 1201, "step_time": 77.11000019637868 }, { "clip_ratio/high_max": 0.006409508700016886, "clip_ratio/high_mean": 0.0012819017574656755, "clip_ratio/low_mean": 0.0011230468899157131, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024049486237345263, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 592.0, "completions/mean_length": 966.9000244140625, "completions/mean_terminated_length": 567.2000122070312, "completions/min_length": 513.0, "completions/min_terminated_length": 513.0, "entropy": 0.06368368491530418, "epoch": 0.6837315130830489, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.05943909287452698, "kl": 0.038678440963849425, "learning_rate": 2.8515358361774743e-06, "loss": -0.0114, "num_tokens": 126968173.0, "reward": 0.17499999701976776, "reward_std": 0.5006406307220459, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.05000000074505806, "rewards/gpt_judge_reward/std": 0.22072142362594604, "sampling/importance_sampling_ratio/max": 1.8654570579528809, "sampling/importance_sampling_ratio/mean": 0.4359722137451172, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7131435871124268, "sampling/sampling_logp_difference/mean": 0.00896645151078701, "step": 1202, "step_time": 75.36600040039048 }, { "clip_ratio/high_max": 0.007937006128486246, "clip_ratio/high_mean": 0.0021710123764933087, "clip_ratio/low_mean": 0.0031005859200377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005271598231047392, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 977.0, "completions/mean_length": 953.0499877929688, "completions/mean_terminated_length": 618.5714721679688, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "entropy": 0.14375479985028505, "epoch": 0.6843003412969283, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.009114536456763744, "kl": 0.011959602008573711, "learning_rate": 2.8464163822525597e-06, "loss": 0.0102, "num_tokens": 127079959.0, "reward": 0.3499999940395355, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.167191505432129, "sampling/importance_sampling_ratio/mean": 0.22003455460071564, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.314084053039551, "sampling/sampling_logp_difference/mean": 0.015132454223930836, "step": 1203, "step_time": 76.74810688011348 }, { "clip_ratio/high_max": 0.010824090568348765, "clip_ratio/high_mean": 0.0038320699968608096, "clip_ratio/low_mean": 0.0012939453008584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005126015224959701, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 930.0, "completions/mean_length": 791.375, "completions/mean_terminated_length": 619.434814453125, "completions/min_length": 432.0, "completions/min_terminated_length": 432.0, "entropy": 0.08342871256172657, "epoch": 0.6848691695108078, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.016409657895565033, "kl": 0.013844507746398449, "learning_rate": 2.8412969283276447e-06, "loss": -0.0207, "num_tokens": 127187654.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.2696826457977295, "sampling/importance_sampling_ratio/mean": 0.34372425079345703, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3554396629333496, "sampling/sampling_logp_difference/mean": 0.01239710208028555, "step": 1204, "step_time": 76.59345134161413 }, { "clip_ratio/high_max": 0.010196437069680542, "clip_ratio/high_mean": 0.002430093416478485, "clip_ratio/low_mean": 0.0014648437609139364, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038949370937189087, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 786.4000244140625, "completions/mean_terminated_length": 496.0, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.10876643192023039, "epoch": 0.6854379977246872, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03160276636481285, "kl": 0.013259710627608001, "learning_rate": 2.8361774744027306e-06, "loss": -0.0239, "num_tokens": 127288022.0, "reward": 0.9000000357627869, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.780750274658203, "sampling/importance_sampling_ratio/mean": 0.39175543189048767, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.080096960067749, "sampling/sampling_logp_difference/mean": 0.013873370364308357, "step": 1205, "step_time": 71.29853401100263 }, { "clip_ratio/high_max": 0.0025483123608864844, "clip_ratio/high_mean": 0.0005096624736324884, "clip_ratio/low_mean": 0.0008300781410071068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013397406146395952, "completions/clipped_ratio": 0.949999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 897.0, "completions/mean_length": 1017.2000122070312, "completions/mean_terminated_length": 888.0, "completions/min_length": 879.0, "completions/min_terminated_length": 879.0, "entropy": 0.10851031355559826, "epoch": 0.6860068259385665, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.014181091450154781, "kl": 0.007933349872473627, "learning_rate": 2.8310580204778156e-06, "loss": -0.0384, "num_tokens": 127398734.0, "reward": 0.05000000074505806, "reward_std": 0.22072142362594604, "rewards/format_reward/mean": 0.05000000074505806, "rewards/format_reward/std": 0.22072142362594604, "rewards/gpt_judge_reward/mean": 0.0, "rewards/gpt_judge_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.4323461055755615, "sampling/importance_sampling_ratio/mean": 0.4709452986717224, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5218229293823242, "sampling/sampling_logp_difference/mean": 0.011806781403720379, "step": 1206, "step_time": 73.3930360279046 }, { "clip_ratio/high_max": 0.01402848947327584, "clip_ratio/high_mean": 0.005209221395489294, "clip_ratio/low_mean": 0.0015869140570430318, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006796135538024828, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 844.625, "completions/mean_terminated_length": 625.388916015625, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.1377064771950245, "epoch": 0.6865756541524459, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03028525970876217, "kl": 0.015366594307124615, "learning_rate": 2.825938566552901e-06, "loss": -0.0438, "num_tokens": 127509359.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.7389845848083496, "sampling/importance_sampling_ratio/mean": 0.4250718653202057, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.648665428161621, "sampling/sampling_logp_difference/mean": 0.015341385267674923, "step": 1207, "step_time": 74.73385626729578 }, { "clip_ratio/high_max": 0.015639795339666307, "clip_ratio/high_mean": 0.004532109334832057, "clip_ratio/low_mean": 0.0015136719011934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006045781308785081, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 978.0, "completions/mean_length": 761.75, "completions/mean_terminated_length": 567.9130249023438, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.12089407816529274, "epoch": 0.6871444823663254, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.058915846049785614, "kl": 0.01379763160366565, "learning_rate": 2.8208191126279865e-06, "loss": -0.0264, "num_tokens": 127617197.0, "reward": 1.100000023841858, "reward_std": 0.9818872213363647, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057363510131836, "sampling/importance_sampling_ratio/max": 2.312544107437134, "sampling/importance_sampling_ratio/mean": 0.6286109685897827, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9847967624664307, "sampling/sampling_logp_difference/mean": 0.014939061366021633, "step": 1208, "step_time": 73.02034601662308 }, { "clip_ratio/high_max": 0.015333821764215827, "clip_ratio/high_mean": 0.0058921489981003106, "clip_ratio/low_mean": 0.0021240234054857865, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008016172505449504, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 841.2000122070312, "completions/mean_terminated_length": 617.7777709960938, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.16400541178882122, "epoch": 0.6877133105802048, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.008945240639150143, "kl": 0.013041858794167638, "learning_rate": 2.815699658703072e-06, "loss": -0.0031, "num_tokens": 127728845.0, "reward": 0.699999988079071, "reward_std": 0.8533493876457214, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.089428663253784, "sampling/importance_sampling_ratio/mean": 0.2835003733634949, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.4265360832214355, "sampling/sampling_logp_difference/mean": 0.017294028773903847, "step": 1209, "step_time": 75.87279102089815 }, { "clip_ratio/high_max": 0.0032989438623189926, "clip_ratio/high_mean": 0.0007649781182408333, "clip_ratio/low_mean": 0.003124999988358468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038899781357031316, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 913.0, "completions/mean_length": 928.8250122070312, "completions/mean_terminated_length": 677.9091186523438, "completions/min_length": 575.0, "completions/min_terminated_length": 575.0, "entropy": 0.12400416936725378, "epoch": 0.6882821387940842, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019230209290981293, "kl": 0.010955098667182028, "learning_rate": 2.810580204778157e-06, "loss": 0.0187, "num_tokens": 127837342.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.479454278945923, "sampling/importance_sampling_ratio/mean": 0.4300681948661804, "sampling/importance_sampling_ratio/min": 0.006664319895207882, "sampling/sampling_logp_difference/max": 1.7084957361221313, "sampling/sampling_logp_difference/mean": 0.013574287295341492, "step": 1210, "step_time": 74.13930459157564 }, { "clip_ratio/high_max": 0.018789921537972987, "clip_ratio/high_mean": 0.005446856943308376, "clip_ratio/low_mean": 0.002880859319702722, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008327716146595776, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 887.6500244140625, "completions/mean_terminated_length": 660.4000244140625, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.11826269049197435, "epoch": 0.6888509670079636, "frac_reward_zero_std": 0.0, "grad_norm": 0.027047203853726387, "kl": 0.013111089123412967, "learning_rate": 2.8054607508532423e-06, "loss": 0.0106, "num_tokens": 127935208.0, "reward": 0.675000011920929, "reward_std": 0.9167249202728271, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.4502182006835938, "sampling/importance_sampling_ratio/mean": 0.34518834948539734, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.096489906311035, "sampling/sampling_logp_difference/mean": 0.013892230577766895, "step": 1211, "step_time": 71.6206147801131 }, { "clip_ratio/high_max": 0.010244653734844178, "clip_ratio/high_mean": 0.002887857161113061, "clip_ratio/low_mean": 0.0015136718648136593, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004401528974995017, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 848.0, "completions/mean_length": 923.7000122070312, "completions/mean_terminated_length": 622.7999877929688, "completions/min_length": 445.0, "completions/min_terminated_length": 445.0, "entropy": 0.09102311544120312, "epoch": 0.689419795221843, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02907886914908886, "kl": 0.008700825739651918, "learning_rate": 2.8003412969283278e-06, "loss": 0.0291, "num_tokens": 128040924.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.3977057933807373, "sampling/importance_sampling_ratio/mean": 0.5712910890579224, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5902409553527832, "sampling/sampling_logp_difference/mean": 0.011570446193218231, "step": 1212, "step_time": 75.62463085190393 }, { "clip_ratio/high_max": 0.021406386978924274, "clip_ratio/high_mean": 0.006759818526916206, "clip_ratio/low_mean": 0.0008233035332523286, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007583122176583856, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 700.3250122070312, "completions/mean_terminated_length": 544.4815063476562, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.12105398159474134, "epoch": 0.6899886234357224, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.020071443170309067, "kl": 0.015849547344259918, "learning_rate": 2.795221843003413e-06, "loss": 0.0231, "num_tokens": 128135361.0, "reward": 1.274999976158142, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.7233147621154785, "sampling/importance_sampling_ratio/mean": 0.3588976263999939, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.417466640472412, "sampling/sampling_logp_difference/mean": 0.015981122851371765, "step": 1213, "step_time": 68.72871363372542 }, { "clip_ratio/high_max": 0.0075372952269390225, "clip_ratio/high_mean": 0.0015074590628501028, "clip_ratio/low_mean": 0.0012207031395519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002728162129642442, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 652.1500244140625, "completions/mean_terminated_length": 511.10345458984375, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.12854814156889915, "epoch": 0.6905574516496018, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011237415485084057, "kl": 0.015319300349801779, "learning_rate": 2.790102389078498e-06, "loss": 0.0304, "num_tokens": 128241695.0, "reward": 1.4500000476837158, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.7250000238418579, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.8806560039520264, "sampling/importance_sampling_ratio/mean": 0.5474169850349426, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.500349521636963, "sampling/sampling_logp_difference/mean": 0.017540639266371727, "step": 1214, "step_time": 75.37974404729903 }, { "clip_ratio/high_max": 0.015396042028442025, "clip_ratio/high_mean": 0.0039356709603453055, "clip_ratio/low_mean": 0.0017643449828028679, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005700015841284767, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 893.2750244140625, "completions/mean_terminated_length": 650.5, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.11004889104515314, "epoch": 0.6911262798634812, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013117486611008644, "kl": 0.011305490974336863, "learning_rate": 2.7849829351535836e-06, "loss": -0.0074, "num_tokens": 128350666.0, "reward": 0.625, "reward_std": 0.896932065486908, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 1.5209439992904663, "sampling/importance_sampling_ratio/mean": 0.2739993929862976, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.675074815750122, "sampling/sampling_logp_difference/mean": 0.013649349100887775, "step": 1215, "step_time": 73.42915872554295 }, { "clip_ratio/high_max": 0.006275933643337339, "clip_ratio/high_mean": 0.0014078585081733763, "clip_ratio/low_mean": 0.0016390200798923615, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030468786135315895, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 859.4000244140625, "completions/mean_terminated_length": 658.2222290039062, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "entropy": 0.10673116892576218, "epoch": 0.6916951080773607, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.028579212725162506, "kl": 0.011636888608336449, "learning_rate": 2.779863481228669e-06, "loss": 0.0948, "num_tokens": 128460938.0, "reward": 0.800000011920929, "reward_std": 0.9391758441925049, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.854780673980713, "sampling/importance_sampling_ratio/mean": 0.45878809690475464, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.118818759918213, "sampling/sampling_logp_difference/mean": 0.012521679513156414, "step": 1216, "step_time": 73.92428757017478 }, { "clip_ratio/high_max": 0.010751593741588295, "clip_ratio/high_mean": 0.0031198598589980975, "clip_ratio/low_mean": 0.0015624999832652975, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004682359722210094, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 855.4500122070312, "completions/mean_terminated_length": 649.4444580078125, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.10474005248397589, "epoch": 0.6922639362912401, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025303320959210396, "kl": 0.011988902930170298, "learning_rate": 2.7747440273037545e-06, "loss": -0.0256, "num_tokens": 128572172.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.9559623003005981, "sampling/importance_sampling_ratio/mean": 0.46885809302330017, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9855390787124634, "sampling/sampling_logp_difference/mean": 0.012734907679259777, "step": 1217, "step_time": 75.33573001646437 }, { "clip_ratio/high_max": 0.011562586965737864, "clip_ratio/high_mean": 0.00493386916059535, "clip_ratio/low_mean": 0.0014160156169964466, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006349884672090411, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 947.0, "completions/mean_length": 811.0499877929688, "completions/mean_terminated_length": 598.1000366210938, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "entropy": 0.08808539435267448, "epoch": 0.6928327645051194, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03156276419758797, "kl": 0.01154649571981281, "learning_rate": 2.7696245733788395e-06, "loss": 0.0638, "num_tokens": 128676022.0, "reward": 0.800000011920929, "reward_std": 0.8828856945037842, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.7734899520874023, "sampling/importance_sampling_ratio/mean": 0.3560999631881714, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.771752119064331, "sampling/sampling_logp_difference/mean": 0.012139013968408108, "step": 1218, "step_time": 73.82440176885575 }, { "clip_ratio/high_max": 0.012024737079627812, "clip_ratio/high_mean": 0.0029629737546201795, "clip_ratio/low_mean": 0.001367187505820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004330161274992861, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 939.5499877929688, "completions/mean_terminated_length": 716.9091186523438, "completions/min_length": 501.0, "completions/min_terminated_length": 501.0, "entropy": 0.08042849786579609, "epoch": 0.6934015927189988, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02108648791909218, "kl": 0.009223178843967617, "learning_rate": 2.764505119453925e-06, "loss": -0.0392, "num_tokens": 128779844.0, "reward": 0.4749999940395355, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.7797083854675293, "sampling/importance_sampling_ratio/mean": 0.5098270177841187, "sampling/importance_sampling_ratio/min": 0.00601355591788888, "sampling/sampling_logp_difference/max": 1.7735861539840698, "sampling/sampling_logp_difference/mean": 0.011090616695582867, "step": 1219, "step_time": 71.92666405322962 }, { "clip_ratio/high_max": 0.00908434996381402, "clip_ratio/high_mean": 0.0022100814676377922, "clip_ratio/low_mean": 0.001635742184589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00384582371043507, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 872.5, "completions/mean_terminated_length": 620.0000610351562, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "entropy": 0.06413777498528361, "epoch": 0.6939704209328783, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01545746624469757, "kl": 0.008223684562835842, "learning_rate": 2.7593856655290104e-06, "loss": 0.0605, "num_tokens": 128878888.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.089867353439331, "sampling/importance_sampling_ratio/mean": 0.46386975049972534, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5948915481567383, "sampling/sampling_logp_difference/mean": 0.009478755295276642, "step": 1220, "step_time": 69.21384461782873 }, { "clip_ratio/high_max": 0.005808497371617705, "clip_ratio/high_mean": 0.0011863057734444737, "clip_ratio/low_mean": 0.001806640655559022, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002992946421727538, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 742.875, "completions/mean_terminated_length": 488.5238037109375, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.07730844942852855, "epoch": 0.6945392491467577, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02474348433315754, "kl": 0.012636626313906163, "learning_rate": 2.754266211604096e-06, "loss": 0.0903, "num_tokens": 128978531.0, "reward": 1.024999976158142, "reward_std": 0.9996795058250427, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.9279749393463135, "sampling/importance_sampling_ratio/mean": 0.5151351094245911, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5766563415527344, "sampling/sampling_logp_difference/mean": 0.01180759072303772, "step": 1221, "step_time": 72.79721842124127 }, { "clip_ratio/high_max": 0.006592329475097358, "clip_ratio/high_mean": 0.0017860271036624908, "clip_ratio/low_mean": 0.0015380859476863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033241130877286196, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 827.0, "completions/mean_length": 870.2750244140625, "completions/mean_terminated_length": 551.0, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.10813913214951754, "epoch": 0.6951080773606371, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04168281331658363, "kl": 0.011273706331849098, "learning_rate": 2.749146757679181e-06, "loss": -0.0608, "num_tokens": 129084894.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.7052407264709473, "sampling/importance_sampling_ratio/mean": 0.32621708512306213, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8378232717514038, "sampling/sampling_logp_difference/mean": 0.013456061482429504, "step": 1222, "step_time": 71.08023834927008 }, { "clip_ratio/high_max": 0.0072128939209505916, "clip_ratio/high_mean": 0.0016859070456121117, "clip_ratio/low_mean": 0.0013916016014263732, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030775086488574743, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 914.0, "completions/mean_length": 898.9750366210938, "completions/mean_terminated_length": 666.7857666015625, "completions/min_length": 468.0, "completions/min_terminated_length": 468.0, "entropy": 0.08774658478796482, "epoch": 0.6956769055745164, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04476700350642204, "kl": 0.008328184427227825, "learning_rate": 2.7440273037542663e-06, "loss": -0.0281, "num_tokens": 129187797.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.7270185947418213, "sampling/importance_sampling_ratio/mean": 0.42840248346328735, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.089491367340088, "sampling/sampling_logp_difference/mean": 0.010664490051567554, "step": 1223, "step_time": 71.72188133443706 }, { "clip_ratio/high_max": 0.007340600277530029, "clip_ratio/high_mean": 0.002003241424972657, "clip_ratio/low_mean": 0.0020507812732830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004054022661875933, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 911.0, "completions/mean_length": 898.4249877929688, "completions/mean_terminated_length": 637.6154174804688, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.11239809356629848, "epoch": 0.6962457337883959, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02882172167301178, "kl": 0.010781048040371388, "learning_rate": 2.7389078498293517e-06, "loss": 0.034, "num_tokens": 129297886.0, "reward": 0.5, "reward_std": 0.7844645380973816, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.7445926666259766, "sampling/importance_sampling_ratio/mean": 0.48255157470703125, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.346175193786621, "sampling/sampling_logp_difference/mean": 0.013963162899017334, "step": 1224, "step_time": 76.09294971101917 }, { "clip_ratio/high_max": 0.008642637287266552, "clip_ratio/high_mean": 0.002052761730737984, "clip_ratio/low_mean": 0.0016945012248470448, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037472629483090714, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 917.125, "completions/mean_terminated_length": 635.3636474609375, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.1056671142578125, "epoch": 0.6968145620022753, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02084108255803585, "kl": 0.010292094200849533, "learning_rate": 2.733788395904437e-06, "loss": -0.0193, "num_tokens": 129399771.0, "reward": 0.45000001788139343, "reward_std": 0.7828285694122314, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.4192521572113037, "sampling/importance_sampling_ratio/mean": 0.43096551299095154, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7933061122894287, "sampling/sampling_logp_difference/mean": 0.0116175077855587, "step": 1225, "step_time": 73.01770936464891 }, { "clip_ratio/high_max": 0.02108709537424147, "clip_ratio/high_mean": 0.008809864113572985, "clip_ratio/low_mean": 0.002783203177386895, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.011593067436479032, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 805.1749877929688, "completions/mean_terminated_length": 643.434814453125, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.14412256702780724, "epoch": 0.6973833902161547, "frac_reward_zero_std": 0.0, "grad_norm": 0.03608948364853859, "kl": 0.013965475489385426, "learning_rate": 2.728668941979522e-06, "loss": 0.0114, "num_tokens": 129502226.0, "reward": 1.125, "reward_std": 0.9919548630714417, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.8740363121032715, "sampling/importance_sampling_ratio/mean": 0.367761492729187, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.269232749938965, "sampling/sampling_logp_difference/mean": 0.01702558435499668, "step": 1226, "step_time": 74.21241144347005 }, { "clip_ratio/high_max": 0.002604122390039265, "clip_ratio/high_mean": 0.0005208244838286191, "clip_ratio/low_mean": 0.0012451171787688509, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017659416116657667, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 931.2750244140625, "completions/mean_terminated_length": 653.1000366210938, "completions/min_length": 381.0, "completions/min_terminated_length": 381.0, "entropy": 0.10297138895839453, "epoch": 0.6979522184300341, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.008354980498552322, "kl": 0.014291423838585615, "learning_rate": 2.723549488054608e-06, "loss": 0.0624, "num_tokens": 129614613.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.9122216701507568, "sampling/importance_sampling_ratio/mean": 0.35860586166381836, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.313910961151123, "sampling/sampling_logp_difference/mean": 0.012654547579586506, "step": 1227, "step_time": 76.15206751925871 }, { "clip_ratio/high_max": 0.003313006993266754, "clip_ratio/high_mean": 0.0006626013928325847, "clip_ratio/low_mean": 0.00019531250291038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008579138957429677, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 1016.0, "completions/mean_terminated_length": 978.2857666015625, "completions/min_length": 910.0, "completions/min_terminated_length": 910.0, "entropy": 0.03834734531119466, "epoch": 0.6985210466439136, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.01859941892325878, "kl": 0.006706840940751135, "learning_rate": 2.718430034129693e-06, "loss": 0.0463, "num_tokens": 129722613.0, "reward": 0.3499999940395355, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.121983051300049, "sampling/importance_sampling_ratio/mean": 0.4127011299133301, "sampling/importance_sampling_ratio/min": 0.0019319017883390188, "sampling/sampling_logp_difference/max": 1.5228829383850098, "sampling/sampling_logp_difference/mean": 0.006454445421695709, "step": 1228, "step_time": 75.56085870577954 }, { "clip_ratio/high_max": 0.010181126184761524, "clip_ratio/high_mean": 0.004235056061588693, "clip_ratio/low_mean": 0.0009291534370277077, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005164209534996189, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 868.75, "completions/mean_terminated_length": 754.0, "completions/min_length": 506.0, "completions/min_terminated_length": 506.0, "entropy": 0.13091673702001572, "epoch": 0.699089874857793, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026041382923722267, "kl": 0.011220738233532757, "learning_rate": 2.7133105802047784e-06, "loss": -0.0522, "num_tokens": 129829379.0, "reward": 0.949999988079071, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.9742143154144287, "sampling/importance_sampling_ratio/mean": 0.5184515118598938, "sampling/importance_sampling_ratio/min": 0.0037809975910931826, "sampling/sampling_logp_difference/max": 1.646198034286499, "sampling/sampling_logp_difference/mean": 0.013763484545052052, "step": 1229, "step_time": 73.7840036698617 }, { "clip_ratio/high_max": 0.010705935943406075, "clip_ratio/high_mean": 0.002714149304665625, "clip_ratio/low_mean": 0.0009277343833673513, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003641883726231754, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 746.4249877929688, "completions/mean_terminated_length": 468.8500061035156, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.08422948326915503, "epoch": 0.6996587030716723, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01632142812013626, "kl": 0.0121874661417678, "learning_rate": 2.7081911262798635e-06, "loss": 0.065, "num_tokens": 129933660.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.605616569519043, "sampling/importance_sampling_ratio/mean": 0.5156399607658386, "sampling/importance_sampling_ratio/min": 0.0032843956723809242, "sampling/sampling_logp_difference/max": 1.4947876930236816, "sampling/sampling_logp_difference/mean": 0.012096116319298744, "step": 1230, "step_time": 70.45558857033029 }, { "clip_ratio/high_max": 0.016977531253360212, "clip_ratio/high_mean": 0.004686634027166292, "clip_ratio/low_mean": 0.0017578125116415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006444446567911655, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 745.625, "completions/mean_terminated_length": 640.0344848632812, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.1343091893941164, "epoch": 0.7002275312855518, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.050345417112112045, "kl": 0.015695739188231528, "learning_rate": 2.7030716723549493e-06, "loss": -0.0959, "num_tokens": 130038269.0, "reward": 1.4500000476837158, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.7250000238418579, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.9606138467788696, "sampling/importance_sampling_ratio/mean": 0.3515223264694214, "sampling/importance_sampling_ratio/min": 0.006128090899437666, "sampling/sampling_logp_difference/max": 2.075733184814453, "sampling/sampling_logp_difference/mean": 0.016167977824807167, "step": 1231, "step_time": 73.17361299390905 }, { "clip_ratio/high_max": 0.007888464780990034, "clip_ratio/high_mean": 0.0027658441540552303, "clip_ratio/low_mean": 0.002001953122089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004767797247041017, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 893.1500244140625, "completions/mean_terminated_length": 675.0667114257812, "completions/min_length": 427.0, "completions/min_terminated_length": 427.0, "entropy": 0.10804736847057939, "epoch": 0.7007963594994312, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02191132865846157, "kl": 0.011875141237396747, "learning_rate": 2.6979522184300343e-06, "loss": 0.062, "num_tokens": 130144675.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.8930530548095703, "sampling/importance_sampling_ratio/mean": 0.4037114679813385, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0626864433288574, "sampling/sampling_logp_difference/mean": 0.012438821606338024, "step": 1232, "step_time": 74.68991632922553 }, { "clip_ratio/high_max": 0.015337958931922913, "clip_ratio/high_mean": 0.007095634617144242, "clip_ratio/low_mean": 0.0016778232202341314, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008773458073846996, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 802.5250244140625, "completions/mean_terminated_length": 683.2692260742188, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.12348593678325415, "epoch": 0.7013651877133106, "frac_reward_zero_std": 0.0, "grad_norm": 0.024498702958226204, "kl": 0.011363920115400106, "learning_rate": 2.6928327645051193e-06, "loss": -0.0193, "num_tokens": 130251024.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.3033134937286377, "sampling/importance_sampling_ratio/mean": 0.4413414001464844, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9859228134155273, "sampling/sampling_logp_difference/mean": 0.01467839814722538, "step": 1233, "step_time": 73.19546806509607 }, { "clip_ratio/high_max": 0.016498018929269165, "clip_ratio/high_mean": 0.005418391112470999, "clip_ratio/low_mean": 0.0006871520836284617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006105543317971751, "completions/clipped_ratio": 0.125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 655.2750244140625, "completions/mean_terminated_length": 602.5999755859375, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.1261386964470148, "epoch": 0.70193401592719, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04573436453938484, "kl": 0.014566892525181174, "learning_rate": 2.6877133105802048e-06, "loss": 0.1169, "num_tokens": 130346395.0, "reward": 1.6750000715255737, "reward_std": 0.6938373446464539, "rewards/format_reward/mean": 0.875, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.800000011920929, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.9967615604400635, "sampling/importance_sampling_ratio/mean": 0.48720455169677734, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0742733478546143, "sampling/sampling_logp_difference/mean": 0.015880081802606583, "step": 1234, "step_time": 69.71136174513958 }, { "clip_ratio/high_max": 0.015318713732995093, "clip_ratio/high_mean": 0.005075819848570973, "clip_ratio/low_mean": 0.0017137960421678144, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0067896159598603845, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 731.2750244140625, "completions/mean_terminated_length": 620.2413940429688, "completions/min_length": 429.0, "completions/min_terminated_length": 429.0, "entropy": 0.11016166117042303, "epoch": 0.7025028441410694, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02588452771306038, "kl": 0.015052083530463278, "learning_rate": 2.6825938566552898e-06, "loss": 0.0176, "num_tokens": 130441678.0, "reward": 1.3250000476837158, "reward_std": 0.8589648008346558, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.614081859588623, "sampling/importance_sampling_ratio/mean": 0.5585082173347473, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.216035842895508, "sampling/sampling_logp_difference/mean": 0.014442652463912964, "step": 1235, "step_time": 69.44595001917332 }, { "clip_ratio/high_max": 0.01637331664096564, "clip_ratio/high_mean": 0.004638717480702326, "clip_ratio/low_mean": 0.0024595467766630463, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007098264235537499, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 878.5250244140625, "completions/mean_terminated_length": 660.3125, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "entropy": 0.10885521303862333, "epoch": 0.7030716723549488, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.010950904339551926, "kl": 0.015331625356338918, "learning_rate": 2.6774744027303756e-06, "loss": 0.0379, "num_tokens": 130544467.0, "reward": 0.6500000357627869, "reward_std": 0.8638019561767578, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 0.8787669539451599, "sampling/importance_sampling_ratio/mean": 0.18293841183185577, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.595069408416748, "sampling/sampling_logp_difference/mean": 0.013790510594844818, "step": 1236, "step_time": 72.85678410623223 }, { "clip_ratio/high_max": 0.010021433641668409, "clip_ratio/high_mean": 0.004436668968992308, "clip_ratio/low_mean": 0.001262781719560735, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005699450732208788, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 765.2999877929688, "completions/mean_terminated_length": 626.0, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.0898198545910418, "epoch": 0.7036405005688282, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019935134798288345, "kl": 0.012841391609981656, "learning_rate": 2.6723549488054606e-06, "loss": 0.0143, "num_tokens": 130636671.0, "reward": 1.100000023841858, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.544344425201416, "sampling/importance_sampling_ratio/mean": 0.45202580094337463, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0745108127593994, "sampling/sampling_logp_difference/mean": 0.012887558899819851, "step": 1237, "step_time": 68.82187533285469 }, { "clip_ratio/high_max": 0.011939848365727812, "clip_ratio/high_mean": 0.0038600352709181607, "clip_ratio/low_mean": 0.0016902198549360037, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005550255154957995, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 903.75, "completions/mean_terminated_length": 680.4285888671875, "completions/min_length": 436.0, "completions/min_terminated_length": 436.0, "entropy": 0.08888598904013634, "epoch": 0.7042093287827076, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01650337129831314, "kl": 0.011502858716994524, "learning_rate": 2.667235494880546e-06, "loss": 0.0656, "num_tokens": 130743869.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.884232759475708, "sampling/importance_sampling_ratio/mean": 0.5303648114204407, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.24228835105896, "sampling/sampling_logp_difference/mean": 0.011712255887687206, "step": 1238, "step_time": 73.36673624557443 }, { "clip_ratio/high_max": 0.015898587298579514, "clip_ratio/high_mean": 0.004863693029619753, "clip_ratio/low_mean": 0.001245117206053692, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00610881025204435, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 770.7250366210938, "completions/mean_terminated_length": 563.5, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.10241761710494757, "epoch": 0.7047781569965871, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01590576022863388, "kl": 0.013719977054279298, "learning_rate": 2.662116040955631e-06, "loss": -0.0194, "num_tokens": 130848466.0, "reward": 1.100000023841858, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.726304054260254, "sampling/importance_sampling_ratio/mean": 0.4179038107395172, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2234268188476562, "sampling/sampling_logp_difference/mean": 0.013335660099983215, "step": 1239, "step_time": 72.25742623722181 }, { "clip_ratio/high_max": 0.004653589072404429, "clip_ratio/high_mean": 0.0010134992699022405, "clip_ratio/low_mean": 0.0016601562820142135, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002673655573744327, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 996.2999877929688, "completions/mean_terminated_length": 802.4000244140625, "completions/min_length": 604.0, "completions/min_terminated_length": 604.0, "entropy": 0.07731362571939826, "epoch": 0.7053469852104665, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.048687733709812164, "kl": 0.00858955824514851, "learning_rate": 2.656996587030717e-06, "loss": -0.1708, "num_tokens": 130958174.0, "reward": 0.25, "reward_std": 0.6698641180992126, "rewards/format_reward/mean": 0.125, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.8189892768859863, "sampling/importance_sampling_ratio/mean": 0.5284326076507568, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.600822687149048, "sampling/sampling_logp_difference/mean": 0.009707983583211899, "step": 1240, "step_time": 75.53785404888913 }, { "clip_ratio/high_max": 0.012235037283971906, "clip_ratio/high_mean": 0.005143757967744023, "clip_ratio/low_mean": 0.0020366951066534966, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007180453161709011, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 929.0, "completions/mean_length": 791.0250244140625, "completions/mean_terminated_length": 600.4091186523438, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.07845282088965178, "epoch": 0.7059158134243458, "frac_reward_zero_std": 0.0, "grad_norm": 0.029205022379755974, "kl": 0.011143061972688884, "learning_rate": 2.651877133105802e-06, "loss": 0.0468, "num_tokens": 131062447.0, "reward": 1.0750000476837158, "reward_std": 0.9971111416816711, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.910733699798584, "sampling/importance_sampling_ratio/mean": 0.6236114501953125, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3137283325195312, "sampling/sampling_logp_difference/mean": 0.011041444726288319, "step": 1241, "step_time": 72.62569689401425 }, { "clip_ratio/high_max": 0.0158234623959288, "clip_ratio/high_mean": 0.006895414320752025, "clip_ratio/low_mean": 0.0015430146268045064, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008438429096713662, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 787.9249877929688, "completions/mean_terminated_length": 674.25927734375, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.09809538954868913, "epoch": 0.7064846416382252, "frac_reward_zero_std": 0.0, "grad_norm": 0.012820644304156303, "kl": 0.013698349648620933, "learning_rate": 2.6467576791808874e-06, "loss": -0.0068, "num_tokens": 131162780.0, "reward": 1.3250000476837158, "reward_std": 0.9442809820175171, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.762060284614563, "sampling/importance_sampling_ratio/mean": 0.24071970582008362, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5951013565063477, "sampling/sampling_logp_difference/mean": 0.013057124800980091, "step": 1242, "step_time": 72.93658947153017 }, { "clip_ratio/high_max": 0.011223276029340923, "clip_ratio/high_mean": 0.002708480868022889, "clip_ratio/low_mean": 0.0024761329295870382, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005184613924939185, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 870.5750122070312, "completions/mean_terminated_length": 640.4375, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "entropy": 0.16056142188608646, "epoch": 0.7070534698521047, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.034011758863925934, "kl": 0.012744504492729902, "learning_rate": 2.6416382252559724e-06, "loss": 0.0443, "num_tokens": 131277219.0, "reward": 0.6500000357627869, "reward_std": 0.8638019561767578, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.5687944889068604, "sampling/importance_sampling_ratio/mean": 0.3546391427516937, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.067373037338257, "sampling/sampling_logp_difference/mean": 0.01703593321144581, "step": 1243, "step_time": 81.08847405086271 }, { "clip_ratio/high_max": 0.008191308239474893, "clip_ratio/high_mean": 0.002694146503927186, "clip_ratio/low_mean": 0.002124023449141532, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004818169923964888, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 817.0, "completions/mean_length": 784.6749877929688, "completions/mean_terminated_length": 460.8823547363281, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.0901999855414033, "epoch": 0.7076222980659841, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.034312885254621506, "kl": 0.014937467174604535, "learning_rate": 2.6365187713310583e-06, "loss": 0.0167, "num_tokens": 131377086.0, "reward": 0.800000011920929, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.6393320560455322, "sampling/importance_sampling_ratio/mean": 0.5049046277999878, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4082419872283936, "sampling/sampling_logp_difference/mean": 0.012758875265717506, "step": 1244, "step_time": 71.2560055849608 }, { "clip_ratio/high_max": 0.0119885487947613, "clip_ratio/high_mean": 0.004168793064309284, "clip_ratio/low_mean": 0.0014648437536379788, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0056336368434131145, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 924.0, "completions/mean_length": 747.6500244140625, "completions/mean_terminated_length": 563.4166870117188, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.10805414151400328, "epoch": 0.7081911262798635, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0380447618663311, "kl": 0.016402926994487643, "learning_rate": 2.6313993174061433e-06, "loss": -0.0885, "num_tokens": 131477328.0, "reward": 1.2000000476837158, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.0561609268188477, "sampling/importance_sampling_ratio/mean": 0.3867320120334625, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5029494762420654, "sampling/sampling_logp_difference/mean": 0.0142094437032938, "step": 1245, "step_time": 129.21095168218017 }, { "clip_ratio/high_max": 0.01568827306618914, "clip_ratio/high_mean": 0.0035392610006965697, "clip_ratio/low_mean": 0.0014160156497382559, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004955276643158868, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 971.0, "completions/mean_length": 892.9249877929688, "completions/mean_terminated_length": 587.0833740234375, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.08538457285612822, "epoch": 0.7087599544937428, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014930155128240585, "kl": 0.012924617622047663, "learning_rate": 2.6262798634812287e-06, "loss": 0.0177, "num_tokens": 131589653.0, "reward": 0.5, "reward_std": 0.8164966106414795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.9097940921783447, "sampling/importance_sampling_ratio/mean": 0.41739076375961304, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.574510335922241, "sampling/sampling_logp_difference/mean": 0.01143328845500946, "step": 1246, "step_time": 74.64365626778454 }, { "clip_ratio/high_max": 0.010955820907838643, "clip_ratio/high_mean": 0.003947723540477455, "clip_ratio/low_mean": 0.0017822265726863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0057299500913359225, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 739.1500244140625, "completions/mean_terminated_length": 528.6087036132812, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.10737085901200771, "epoch": 0.7093287827076223, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.045789685100317, "kl": 0.012900888803415, "learning_rate": 2.6211604095563137e-06, "loss": -0.103, "num_tokens": 131691131.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.7002320289611816, "sampling/importance_sampling_ratio/mean": 0.545632004737854, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.837883472442627, "sampling/sampling_logp_difference/mean": 0.014336148276925087, "step": 1247, "step_time": 73.87813948118128 }, { "clip_ratio/high_max": 0.011319792363792658, "clip_ratio/high_mean": 0.0030495587561745197, "clip_ratio/low_mean": 0.0031738280958961695, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006223386764759198, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 944.0, "completions/mean_length": 933.2250366210938, "completions/mean_terminated_length": 620.5555419921875, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "entropy": 0.13388165831565857, "epoch": 0.7098976109215017, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018893513828516006, "kl": 0.011374967056326568, "learning_rate": 2.6160409556313996e-06, "loss": 0.0364, "num_tokens": 131804420.0, "reward": 0.42500001192092896, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.107865571975708, "sampling/importance_sampling_ratio/mean": 0.3500136733055115, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0993099212646484, "sampling/sampling_logp_difference/mean": 0.014617028646171093, "step": 1248, "step_time": 75.14646767359227 }, { "clip_ratio/high_max": 0.013787385250907391, "clip_ratio/high_mean": 0.004193493936327286, "clip_ratio/low_mean": 0.0020506964865489863, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006244190473807976, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 860.4249877929688, "completions/mean_terminated_length": 587.800048828125, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.1255579460412264, "epoch": 0.7104664391353811, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.010738831013441086, "kl": 0.010866682743653655, "learning_rate": 2.6109215017064846e-06, "loss": -0.0285, "num_tokens": 131898645.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.4892441034317017, "sampling/importance_sampling_ratio/mean": 0.3381945490837097, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.419523239135742, "sampling/sampling_logp_difference/mean": 0.013685464859008789, "step": 1249, "step_time": 75.37688106298447 }, { "clip_ratio/high_max": 0.009327238891273737, "clip_ratio/high_mean": 0.0035647991171572357, "clip_ratio/low_mean": 0.00244140625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006006205308949575, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 957.4750366210938, "completions/mean_terminated_length": 833.9285888671875, "completions/min_length": 598.0, "completions/min_terminated_length": 598.0, "entropy": 0.15421386063098907, "epoch": 0.7110352673492605, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.011578142642974854, "kl": 0.011965013574808836, "learning_rate": 2.60580204778157e-06, "loss": 0.0132, "num_tokens": 132010624.0, "reward": 0.550000011920929, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.5004093647003174, "sampling/importance_sampling_ratio/mean": 0.2158813774585724, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7935659885406494, "sampling/sampling_logp_difference/mean": 0.015869345515966415, "step": 1250, "step_time": 75.18309159181081 }, { "clip_ratio/high_max": 0.010482327023055404, "clip_ratio/high_mean": 0.0024480490246787667, "clip_ratio/low_mean": 0.0014892578328726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003937306842999533, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 850.4750366210938, "completions/mean_terminated_length": 590.1875, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "entropy": 0.09025697596371174, "epoch": 0.71160409556314, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.030769167467951775, "kl": 0.014575062552466989, "learning_rate": 2.6006825938566555e-06, "loss": -0.0533, "num_tokens": 132113739.0, "reward": 0.800000011920929, "reward_std": 0.9922778606414795, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.071336269378662, "sampling/importance_sampling_ratio/mean": 0.4327830374240875, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7348970174789429, "sampling/sampling_logp_difference/mean": 0.011843879707157612, "step": 1251, "step_time": 75.34678235184401 }, { "clip_ratio/high_max": 0.013189171615522355, "clip_ratio/high_mean": 0.004018170511699282, "clip_ratio/low_mean": 0.001879158298834227, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005897328752325848, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 949.0, "completions/mean_length": 793.7250366210938, "completions/mean_terminated_length": 539.2105102539062, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.13048970885574818, "epoch": 0.7121729237770194, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024486320093274117, "kl": 0.015170524944551289, "learning_rate": 2.595563139931741e-06, "loss": -0.0908, "num_tokens": 132220776.0, "reward": 0.824999988079071, "reward_std": 0.9306049346923828, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.3305768966674805, "sampling/importance_sampling_ratio/mean": 0.36501985788345337, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7529516220092773, "sampling/sampling_logp_difference/mean": 0.015119544230401516, "step": 1252, "step_time": 75.31695039500482 }, { "clip_ratio/high_max": 0.014070200501009822, "clip_ratio/high_mean": 0.005058998940512538, "clip_ratio/low_mean": 0.0016113280908029992, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006670327013125643, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 960.0, "completions/mean_length": 840.7999877929688, "completions/mean_terminated_length": 592.941162109375, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.09292227029800415, "epoch": 0.7127417519908987, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026938529685139656, "kl": 0.010262880357913673, "learning_rate": 2.590443686006826e-06, "loss": -0.0828, "num_tokens": 132325008.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.995889663696289, "sampling/importance_sampling_ratio/mean": 0.4398861825466156, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2614314556121826, "sampling/sampling_logp_difference/mean": 0.011454702354967594, "step": 1253, "step_time": 74.12594749918208 }, { "clip_ratio/high_max": 0.015160412294790149, "clip_ratio/high_mean": 0.006393229181412607, "clip_ratio/low_mean": 0.002539062472351361, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00893229158828035, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 866.0, "completions/mean_length": 759.8250122070312, "completions/mean_terminated_length": 543.6818237304688, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.14096485264599323, "epoch": 0.7133105802047781, "frac_reward_zero_std": 0.0, "grad_norm": 0.045060381293296814, "kl": 0.016858474351465702, "learning_rate": 2.5853242320819113e-06, "loss": -0.0533, "num_tokens": 132432993.0, "reward": 0.9750000238418579, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.9256954193115234, "sampling/importance_sampling_ratio/mean": 0.6048440933227539, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5466771125793457, "sampling/sampling_logp_difference/mean": 0.016509870067238808, "step": 1254, "step_time": 73.67082192609087 }, { "clip_ratio/high_max": 0.011130219209007919, "clip_ratio/high_mean": 0.0025254450883949175, "clip_ratio/low_mean": 0.0011474609673314262, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003672905935673043, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 858.0, "completions/mean_length": 929.25, "completions/mean_terminated_length": 550.25, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.11613134201616049, "epoch": 0.7138794084186576, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014773202128708363, "kl": 0.010655907331965864, "learning_rate": 2.5802047781569968e-06, "loss": 0.0982, "num_tokens": 132532795.0, "reward": 0.32500001788139343, "reward_std": 0.6938373446464539, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 1.7718756198883057, "sampling/importance_sampling_ratio/mean": 0.3474406599998474, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.560187578201294, "sampling/sampling_logp_difference/mean": 0.011663216166198254, "step": 1255, "step_time": 74.18482593912631 }, { "clip_ratio/high_max": 0.009616252151317894, "clip_ratio/high_mean": 0.0031147657355177216, "clip_ratio/low_mean": 0.002124023434589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005238789191935211, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 902.3500366210938, "completions/mean_terminated_length": 676.4285888671875, "completions/min_length": 502.0, "completions/min_terminated_length": 502.0, "entropy": 0.10847906954586506, "epoch": 0.714448236632537, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026993855834007263, "kl": 0.012642358429729939, "learning_rate": 2.575085324232082e-06, "loss": 0.0575, "num_tokens": 132642601.0, "reward": 0.574999988079071, "reward_std": 0.8439072966575623, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.6757326126098633, "sampling/importance_sampling_ratio/mean": 0.41682711243629456, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6565072536468506, "sampling/sampling_logp_difference/mean": 0.012351540848612785, "step": 1256, "step_time": 77.77145571820438 }, { "clip_ratio/high_max": 0.011029377405066043, "clip_ratio/high_mean": 0.0028526448004413396, "clip_ratio/low_mean": 0.0010742187587311491, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003926863471860997, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 808.0, "completions/mean_length": 815.1500244140625, "completions/mean_terminated_length": 532.5882568359375, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.07907961960881948, "epoch": 0.7150170648464164, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011329244822263718, "kl": 0.015263481123838574, "learning_rate": 2.569965870307167e-06, "loss": 0.0435, "num_tokens": 132756231.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.1406595706939697, "sampling/importance_sampling_ratio/mean": 0.269963800907135, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3279483318328857, "sampling/sampling_logp_difference/mean": 0.010899657383561134, "step": 1257, "step_time": 75.67646558303386 }, { "clip_ratio/high_max": 0.011026937165297568, "clip_ratio/high_mean": 0.002864889509510249, "clip_ratio/low_mean": 0.0014648437390860636, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004329733259510249, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 828.8500366210938, "completions/mean_terminated_length": 564.8235473632812, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.10897350311279297, "epoch": 0.7155858930602957, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.010855542495846748, "kl": 0.010911092977039516, "learning_rate": 2.5648464163822526e-06, "loss": 0.09, "num_tokens": 132867705.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.9413886070251465, "sampling/importance_sampling_ratio/mean": 0.620500385761261, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9860262870788574, "sampling/sampling_logp_difference/mean": 0.013219145126640797, "step": 1258, "step_time": 75.96873430092819 }, { "clip_ratio/high_max": 0.015150844468735158, "clip_ratio/high_mean": 0.005823420826345682, "clip_ratio/low_mean": 0.0010203302881564014, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00684375106357038, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 925.0, "completions/mean_length": 787.4750366210938, "completions/mean_terminated_length": 593.95458984375, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.11884979624301195, "epoch": 0.7161547212741752, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024535702541470528, "kl": 0.013376693532336503, "learning_rate": 2.559726962457338e-06, "loss": 0.0875, "num_tokens": 132977196.0, "reward": 0.9000000357627869, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.1585726737976074, "sampling/importance_sampling_ratio/mean": 0.5586778521537781, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9453225135803223, "sampling/sampling_logp_difference/mean": 0.013861960731446743, "step": 1259, "step_time": 73.97921078582294 }, { "clip_ratio/high_max": 0.008022479276405647, "clip_ratio/high_mean": 0.002118309868819779, "clip_ratio/low_mean": 0.0023800155977369286, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004498325317399576, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 822.1000366210938, "completions/mean_terminated_length": 548.941162109375, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.12077609729021788, "epoch": 0.7167235494880546, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.021897271275520325, "kl": 0.013427366153337061, "learning_rate": 2.5546075085324235e-06, "loss": -0.0403, "num_tokens": 133082152.0, "reward": 0.6000000238418579, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.361990213394165, "sampling/importance_sampling_ratio/mean": 0.36940494179725647, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3139631748199463, "sampling/sampling_logp_difference/mean": 0.014232928864657879, "step": 1260, "step_time": 71.85166831267998 }, { "clip_ratio/high_max": 0.013914098381064832, "clip_ratio/high_mean": 0.005963016301393509, "clip_ratio/low_mean": 0.001061369272065349, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007024385500699282, "completions/clipped_ratio": 0.17499999701976776, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 718.1749877929688, "completions/mean_terminated_length": 653.3030395507812, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "entropy": 0.0952465096488595, "epoch": 0.717292377701934, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022159352898597717, "kl": 0.013768856297247112, "learning_rate": 2.5494880546075085e-06, "loss": -0.0261, "num_tokens": 133181791.0, "reward": 1.625, "reward_std": 0.7741827964782715, "rewards/format_reward/mean": 0.824999988079071, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.800000011920929, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.9900914430618286, "sampling/importance_sampling_ratio/mean": 0.3512178063392639, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1511025428771973, "sampling/sampling_logp_difference/mean": 0.01400185376405716, "step": 1261, "step_time": 70.40136225661263 }, { "clip_ratio/high_max": 0.008449729415588081, "clip_ratio/high_mean": 0.0017784414812922478, "clip_ratio/low_mean": 0.0027099609214928932, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004488402395509183, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 992.625, "completions/mean_terminated_length": 814.8333740234375, "completions/min_length": 565.0, "completions/min_terminated_length": 565.0, "entropy": 0.13216138631105423, "epoch": 0.7178612059158134, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015179703943431377, "kl": 0.012125261360779405, "learning_rate": 2.544368600682594e-06, "loss": 0.1308, "num_tokens": 133288920.0, "reward": 0.20000000298023224, "reward_std": 0.5163977742195129, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.05000000074505806, "rewards/gpt_judge_reward/std": 0.22072142362594604, "sampling/importance_sampling_ratio/max": 1.8667093515396118, "sampling/importance_sampling_ratio/mean": 0.41611558198928833, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3771233558654785, "sampling/sampling_logp_difference/mean": 0.013726040720939636, "step": 1262, "step_time": 74.7770164590329 }, { "clip_ratio/high_max": 0.013615782954730093, "clip_ratio/high_mean": 0.0041222892759833485, "clip_ratio/low_mean": 0.0019531250291038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006075414305087179, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 927.0, "completions/mean_length": 848.2999877929688, "completions/mean_terminated_length": 555.4666748046875, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.10871179774403572, "epoch": 0.7184300341296929, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027066722512245178, "kl": 0.01263155834749341, "learning_rate": 2.5392491467576794e-06, "loss": 0.0749, "num_tokens": 133399420.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.927173137664795, "sampling/importance_sampling_ratio/mean": 0.4971301555633545, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.774524450302124, "sampling/sampling_logp_difference/mean": 0.012898596003651619, "step": 1263, "step_time": 75.09893541992642 }, { "clip_ratio/high_max": 0.012754216731991619, "clip_ratio/high_mean": 0.004492693624342792, "clip_ratio/low_mean": 0.001196289065774181, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00568898266647011, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 836.7999877929688, "completions/mean_terminated_length": 608.0, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.07327198190614581, "epoch": 0.7189988623435722, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023403367027640343, "kl": 0.009994990890845656, "learning_rate": 2.534129692832765e-06, "loss": 0.0798, "num_tokens": 133504228.0, "reward": 0.8500000238418579, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.352738380432129, "sampling/importance_sampling_ratio/mean": 0.44869986176490784, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6617166996002197, "sampling/sampling_logp_difference/mean": 0.01094291266053915, "step": 1264, "step_time": 74.2263277862221 }, { "clip_ratio/high_max": 0.008351680909981951, "clip_ratio/high_mean": 0.001987100338737946, "clip_ratio/low_mean": 0.0015380859476863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003525186257320456, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 904.8500366210938, "completions/mean_terminated_length": 590.727294921875, "completions/min_length": 436.0, "completions/min_terminated_length": 436.0, "entropy": 0.08787054568529129, "epoch": 0.7195676905574516, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.021799733862280846, "kl": 0.01089813548605889, "learning_rate": 2.52901023890785e-06, "loss": -0.0824, "num_tokens": 133612590.0, "reward": 0.42500001192092896, "reward_std": 0.747217059135437, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 1.5788776874542236, "sampling/importance_sampling_ratio/mean": 0.37206965684890747, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9227221012115479, "sampling/sampling_logp_difference/mean": 0.01154126226902008, "step": 1265, "step_time": 75.53216405934654 }, { "clip_ratio/high_max": 0.014935430604964495, "clip_ratio/high_mean": 0.005113393068313599, "clip_ratio/low_mean": 0.0020263672049622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0071397602441720665, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 789.0, "completions/mean_length": 730.9500122070312, "completions/mean_terminated_length": 407.0526428222656, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.08177567645907402, "epoch": 0.7201365187713311, "frac_reward_zero_std": 0.0, "grad_norm": 0.041708990931510925, "kl": 0.014298987342044711, "learning_rate": 2.5238907849829353e-06, "loss": -0.0728, "num_tokens": 133712444.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.3612236976623535, "sampling/importance_sampling_ratio/mean": 0.516508936882019, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.43996262550354, "sampling/sampling_logp_difference/mean": 0.01210204791277647, "step": 1266, "step_time": 72.37625281768851 }, { "clip_ratio/high_max": 0.013405660283751786, "clip_ratio/high_mean": 0.005549087334657088, "clip_ratio/low_mean": 0.0022007786319591105, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007749865879304707, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 953.0, "completions/mean_length": 805.1500244140625, "completions/mean_terminated_length": 626.0909423828125, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "entropy": 0.08478669356554747, "epoch": 0.7207053469852105, "frac_reward_zero_std": 0.0, "grad_norm": 0.027463870123028755, "kl": 0.01332652906421572, "learning_rate": 2.5187713310580207e-06, "loss": 0.0044, "num_tokens": 133820554.0, "reward": 1.0750000476837158, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.883866548538208, "sampling/importance_sampling_ratio/mean": 0.308633029460907, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2330875396728516, "sampling/sampling_logp_difference/mean": 0.01246634591370821, "step": 1267, "step_time": 75.36823676410131 }, { "clip_ratio/high_max": 0.01314700348302722, "clip_ratio/high_mean": 0.0043876316922251135, "clip_ratio/low_mean": 0.0020996094062866177, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006487241218565032, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 908.8250122070312, "completions/mean_terminated_length": 736.0625, "completions/min_length": 403.0, "completions/min_terminated_length": 403.0, "entropy": 0.09553627576678991, "epoch": 0.7212741751990899, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020051315426826477, "kl": 0.010536894667893648, "learning_rate": 2.5136518771331057e-06, "loss": -0.0332, "num_tokens": 133937763.0, "reward": 0.6500000357627869, "reward_std": 0.8638020157814026, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.0256941318511963, "sampling/importance_sampling_ratio/mean": 0.32004696130752563, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3553009033203125, "sampling/sampling_logp_difference/mean": 0.012075910344719887, "step": 1268, "step_time": 77.65144350985065 }, { "clip_ratio/high_max": 0.011395957903005183, "clip_ratio/high_mean": 0.0035527369182091206, "clip_ratio/low_mean": 0.0017036421086231712, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005256379139609635, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 687.6000366210938, "completions/mean_terminated_length": 603.5, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.11176613345742226, "epoch": 0.7218430034129693, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03272901475429535, "kl": 0.015805962379090488, "learning_rate": 2.508532423208191e-06, "loss": -0.0604, "num_tokens": 134044947.0, "reward": 1.4250000715255737, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.2481582164764404, "sampling/importance_sampling_ratio/mean": 0.3933616280555725, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8792037963867188, "sampling/sampling_logp_difference/mean": 0.014725170098245144, "step": 1269, "step_time": 77.35419950820506 }, { "clip_ratio/high_max": 0.014460103819146752, "clip_ratio/high_mean": 0.004977393982699141, "clip_ratio/low_mean": 0.0022577081253984943, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007235102180857211, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 714.1749877929688, "completions/mean_terminated_length": 547.34619140625, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.12448964547365904, "epoch": 0.7224118316268487, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021313581615686417, "kl": 0.015732413507066667, "learning_rate": 2.503412969283276e-06, "loss": 0.0224, "num_tokens": 134147586.0, "reward": 1.100000023841858, "reward_std": 0.9001424908638, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.2742578983306885, "sampling/importance_sampling_ratio/mean": 0.4199508726596832, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.5040740966796875, "sampling/sampling_logp_difference/mean": 0.015318283811211586, "step": 1270, "step_time": 72.1259030373767 }, { "clip_ratio/high_max": 0.01255267660599202, "clip_ratio/high_mean": 0.003961906928452663, "clip_ratio/low_mean": 0.0023193359375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006281242880504578, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 866.8500366210938, "completions/mean_terminated_length": 654.2352905273438, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.09419704508036375, "epoch": 0.7229806598407281, "frac_reward_zero_std": 0.0, "grad_norm": 0.0239646527916193, "kl": 0.01241042127367109, "learning_rate": 2.498293515358362e-06, "loss": 0.0701, "num_tokens": 134239100.0, "reward": 0.8500000238418579, "reward_std": 1.0012813806533813, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.987826347351074, "sampling/importance_sampling_ratio/mean": 0.49753251671791077, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9507555961608887, "sampling/sampling_logp_difference/mean": 0.010738608427345753, "step": 1271, "step_time": 69.05148643814027 }, { "clip_ratio/high_max": 0.012601913011167198, "clip_ratio/high_mean": 0.00320228080090601, "clip_ratio/low_mean": 0.002738447074079886, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005940727860433981, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 765.4500122070312, "completions/mean_terminated_length": 654.6428833007812, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 0.11566019058227539, "epoch": 0.7235494880546075, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.046633314341306686, "kl": 0.02628518594428897, "learning_rate": 2.493174061433447e-06, "loss": -0.1205, "num_tokens": 134340158.0, "reward": 1.149999976158142, "reward_std": 0.8638019561767578, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.815938949584961, "sampling/importance_sampling_ratio/mean": 0.4309470057487488, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.910222053527832, "sampling/sampling_logp_difference/mean": 0.015319282189011574, "step": 1272, "step_time": 72.07457804540172 }, { "clip_ratio/high_max": 0.01117342070210725, "clip_ratio/high_mean": 0.0039912919455673546, "clip_ratio/low_mean": 0.0026611328357830644, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.006652424577623606, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 903.0, "completions/mean_length": 836.625, "completions/mean_terminated_length": 555.5625, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.11545639112591743, "epoch": 0.7241183162684869, "frac_reward_zero_std": 0.0, "grad_norm": 0.03844207152724266, "kl": 0.011471609584987164, "learning_rate": 2.4880546075085325e-06, "loss": 0.0441, "num_tokens": 134440679.0, "reward": 0.625, "reward_std": 0.8378084301948547, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.4353442192077637, "sampling/importance_sampling_ratio/mean": 0.5913631319999695, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0367825031280518, "sampling/sampling_logp_difference/mean": 0.01336708664894104, "step": 1273, "step_time": 73.30118605028838 }, { "clip_ratio/high_max": 0.013715258275624365, "clip_ratio/high_mean": 0.004455265909200534, "clip_ratio/low_mean": 0.0009033203114086064, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00535858616058249, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 690.625, "completions/mean_terminated_length": 417.8636474609375, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 0.09763217438012362, "epoch": 0.7246871444823664, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03230229392647743, "kl": 0.014752939401660115, "learning_rate": 2.4829351535836175e-06, "loss": 0.0098, "num_tokens": 134539912.0, "reward": 1.0750000476837158, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.9868106842041016, "sampling/importance_sampling_ratio/mean": 0.3878037929534912, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7767131328582764, "sampling/sampling_logp_difference/mean": 0.012579909525811672, "step": 1274, "step_time": 70.3409507481847 }, { "clip_ratio/high_max": 0.013168266508728266, "clip_ratio/high_mean": 0.0034819463035091758, "clip_ratio/low_mean": 0.0018006057216553017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005282552010612562, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 902.0, "completions/mean_length": 870.375, "completions/mean_terminated_length": 511.91668701171875, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.12194223329424858, "epoch": 0.7252559726962458, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02740613743662834, "kl": 0.012745829881168902, "learning_rate": 2.4778156996587033e-06, "loss": -0.035, "num_tokens": 134648791.0, "reward": 0.5, "reward_std": 0.8164966106414795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.4847352504730225, "sampling/importance_sampling_ratio/mean": 0.45895129442214966, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.475616455078125, "sampling/sampling_logp_difference/mean": 0.013956022448837757, "step": 1275, "step_time": 74.77907491847873 }, { "clip_ratio/high_max": 0.013763778610154986, "clip_ratio/high_mean": 0.0033895222732098773, "clip_ratio/low_mean": 0.0013427734174911166, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004732295667054132, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 862.0, "completions/mean_length": 834.7999877929688, "completions/mean_terminated_length": 603.5555419921875, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.15162511449307203, "epoch": 0.7258248009101251, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.0065336381085217, "kl": 0.012101785745471716, "learning_rate": 2.4726962457337883e-06, "loss": 0.016, "num_tokens": 134758775.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.7993040084838867, "sampling/importance_sampling_ratio/mean": 0.6172877550125122, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.710012435913086, "sampling/sampling_logp_difference/mean": 0.016268178820610046, "step": 1276, "step_time": 75.19594961567782 }, { "clip_ratio/high_max": 0.009979967959225178, "clip_ratio/high_mean": 0.003172681696014479, "clip_ratio/low_mean": 0.003180768464517314, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006353450124152005, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 712.7000122070312, "completions/mean_terminated_length": 562.8148193359375, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.1642116904258728, "epoch": 0.7263936291240045, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03464134782552719, "kl": 0.018774767988361418, "learning_rate": 2.4675767918088738e-06, "loss": 0.0198, "num_tokens": 134863827.0, "reward": 1.100000023841858, "reward_std": 0.8711913824081421, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.968562126159668, "sampling/importance_sampling_ratio/mean": 0.5123160481452942, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7860842943191528, "sampling/sampling_logp_difference/mean": 0.017044249922037125, "step": 1277, "step_time": 76.1020812059287 }, { "clip_ratio/high_max": 0.0064158926834352314, "clip_ratio/high_mean": 0.0016030228871386498, "clip_ratio/low_mean": 0.0008300781264551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024331010281457566, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 959.0, "completions/mean_length": 832.4249877929688, "completions/mean_terminated_length": 513.1333618164062, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.08430160023272038, "epoch": 0.726962457337884, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.016284996643662453, "kl": 0.010463670245371759, "learning_rate": 2.4624573378839588e-06, "loss": 0.01, "num_tokens": 134968828.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.384051561355591, "sampling/importance_sampling_ratio/mean": 0.5227020382881165, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0569372177124023, "sampling/sampling_logp_difference/mean": 0.011203267611563206, "step": 1278, "step_time": 71.0189924761653 }, { "clip_ratio/high_max": 0.01105444470886141, "clip_ratio/high_mean": 0.004026086025987752, "clip_ratio/low_mean": 0.001042635485646315, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005068721511634067, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 751.25, "completions/mean_terminated_length": 619.9259033203125, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.13702387921512127, "epoch": 0.7275312855517634, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013639899902045727, "kl": 0.012807369232177734, "learning_rate": 2.4573378839590446e-06, "loss": 0.0381, "num_tokens": 135073942.0, "reward": 1.25, "reward_std": 0.9268086552619934, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.168179988861084, "sampling/importance_sampling_ratio/mean": 0.38516363501548767, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0572001934051514, "sampling/sampling_logp_difference/mean": 0.016061073169112206, "step": 1279, "step_time": 73.02348963078111 }, { "clip_ratio/high_max": 0.009918576397467405, "clip_ratio/high_mean": 0.001983715294045396, "clip_ratio/low_mean": 0.0002441406286379788, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002227855922683375, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 842.4249877929688, "completions/mean_terminated_length": 678.1428833007812, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.08131596865132451, "epoch": 0.7281001137656428, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.007098130416125059, "kl": 0.00972320232540369, "learning_rate": 2.4522184300341297e-06, "loss": -0.0353, "num_tokens": 135168855.0, "reward": 0.8500000238418579, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.5842844247817993, "sampling/importance_sampling_ratio/mean": 0.32843369245529175, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9633371829986572, "sampling/sampling_logp_difference/mean": 0.01110411249101162, "step": 1280, "step_time": 72.6545533775352 }, { "clip_ratio/high_max": 0.017023057211190462, "clip_ratio/high_mean": 0.007555174583103508, "clip_ratio/low_mean": 0.0013646570732817054, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008919831598177552, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 842.0, "completions/mean_length": 652.9500122070312, "completions/mean_terminated_length": 493.9285888671875, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.10478801466524601, "epoch": 0.7286689419795221, "frac_reward_zero_std": 0.0, "grad_norm": 0.02940310165286064, "kl": 0.01604063669219613, "learning_rate": 2.447098976109215e-06, "loss": -0.1403, "num_tokens": 135267021.0, "reward": 1.375, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.399831533432007, "sampling/importance_sampling_ratio/mean": 0.4207828640937805, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7955031394958496, "sampling/sampling_logp_difference/mean": 0.014628109522163868, "step": 1281, "step_time": 72.67818771279417 }, { "clip_ratio/high_max": 0.01304149767383933, "clip_ratio/high_mean": 0.004779671304277144, "clip_ratio/low_mean": 0.0015136718666326487, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006293343263678253, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 892.4000244140625, "completions/mean_terminated_length": 695.0, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "entropy": 0.09829786513000727, "epoch": 0.7292377701934016, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03794874995946884, "kl": 0.00976206036284566, "learning_rate": 2.4419795221843e-06, "loss": -0.0594, "num_tokens": 135368205.0, "reward": 0.75, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.902787446975708, "sampling/importance_sampling_ratio/mean": 0.522944986820221, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8139631748199463, "sampling/sampling_logp_difference/mean": 0.011622888967394829, "step": 1282, "step_time": 72.9077840668615 }, { "clip_ratio/high_max": 0.01164837065152824, "clip_ratio/high_mean": 0.002555577695602551, "clip_ratio/low_mean": 0.001220703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037762808351544663, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 640.5499877929688, "completions/mean_terminated_length": 529.2257690429688, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.1369730969890952, "epoch": 0.729806598407281, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.021821051836013794, "kl": 0.014126256806775928, "learning_rate": 2.436860068259386e-06, "loss": -0.0464, "num_tokens": 135461235.0, "reward": 1.524999976158142, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.75, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.2970457077026367, "sampling/importance_sampling_ratio/mean": 0.43338918685913086, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.215940475463867, "sampling/sampling_logp_difference/mean": 0.016102975234389305, "step": 1283, "step_time": 74.53638279507868 }, { "clip_ratio/high_max": 0.014546735677868128, "clip_ratio/high_mean": 0.00378124657436274, "clip_ratio/low_mean": 0.0011962890494032763, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004977535514626652, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 901.7250366210938, "completions/mean_terminated_length": 616.4166870117188, "completions/min_length": 423.0, "completions/min_terminated_length": 423.0, "entropy": 0.07855789829045534, "epoch": 0.7303754266211604, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.021646229550242424, "kl": 0.009613308240659535, "learning_rate": 2.431740614334471e-06, "loss": -0.0672, "num_tokens": 135575744.0, "reward": 0.6000000238418579, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.8939507007598877, "sampling/importance_sampling_ratio/mean": 0.36577603220939636, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.6924526691436768, "sampling/sampling_logp_difference/mean": 0.01162037905305624, "step": 1284, "step_time": 75.38965823082253 }, { "clip_ratio/high_max": 0.016771201393567026, "clip_ratio/high_mean": 0.00523933963268064, "clip_ratio/low_mean": 0.0011962890421273187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0064356287475675344, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 769.0, "completions/mean_length": 867.9500122070312, "completions/mean_terminated_length": 578.1428833007812, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "entropy": 0.1185839120298624, "epoch": 0.7309442548350398, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03684272617101669, "kl": 0.01333158416673541, "learning_rate": 2.4266211604095564e-06, "loss": -0.1612, "num_tokens": 135682238.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.9804751873016357, "sampling/importance_sampling_ratio/mean": 0.4178105890750885, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.462031126022339, "sampling/sampling_logp_difference/mean": 0.014895099215209484, "step": 1285, "step_time": 73.93080648314208 }, { "clip_ratio/high_max": 0.006517958652693778, "clip_ratio/high_mean": 0.0018328521982766688, "clip_ratio/low_mean": 0.001831054705689894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036639069148804992, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 813.9500122070312, "completions/mean_terminated_length": 498.875, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.10002827271819115, "epoch": 0.7315130830489193, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019299598410725594, "kl": 0.012453096336685121, "learning_rate": 2.4215017064846414e-06, "loss": -0.0058, "num_tokens": 135782652.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.255192279815674, "sampling/importance_sampling_ratio/mean": 0.4872780740261078, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1628170013427734, "sampling/sampling_logp_difference/mean": 0.01146022416651249, "step": 1286, "step_time": 72.45888840197586 }, { "clip_ratio/high_max": 0.010947176488116384, "clip_ratio/high_mean": 0.00365452459664084, "clip_ratio/low_mean": 0.0013183594073780114, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004972883936716244, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 806.7999877929688, "completions/mean_terminated_length": 610.2857055664062, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.0940204206854105, "epoch": 0.7320819112627986, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02325419895350933, "kl": 0.011866371147334576, "learning_rate": 2.4163822525597273e-06, "loss": -0.0209, "num_tokens": 135881868.0, "reward": 1.024999976158142, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.5112929344177246, "sampling/importance_sampling_ratio/mean": 0.44871464371681213, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9654403924942017, "sampling/sampling_logp_difference/mean": 0.011821875348687172, "step": 1287, "step_time": 70.93469204730354 }, { "clip_ratio/high_max": 0.012625080766156316, "clip_ratio/high_mean": 0.004029783507576212, "clip_ratio/low_mean": 0.0019042968924622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005934080545557663, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 860.9249877929688, "completions/mean_terminated_length": 589.1333618164062, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.11219177395105362, "epoch": 0.732650739476678, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.05047014728188515, "kl": 0.011118123889900744, "learning_rate": 2.4112627986348123e-06, "loss": -0.0222, "num_tokens": 135987849.0, "reward": 0.7250000238418579, "reward_std": 0.9604353308677673, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.9075448513031006, "sampling/importance_sampling_ratio/mean": 0.5075896978378296, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7038111686706543, "sampling/sampling_logp_difference/mean": 0.013068382628262043, "step": 1288, "step_time": 74.08446981408633 }, { "clip_ratio/high_max": 0.012790920009138063, "clip_ratio/high_mean": 0.004662325845856685, "clip_ratio/low_mean": 0.0007324218677240424, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005394747699028812, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 819.0, "completions/mean_length": 697.375, "completions/mean_terminated_length": 479.625, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.09517090674489737, "epoch": 0.7332195676905574, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022941872477531433, "kl": 0.01318796956911683, "learning_rate": 2.4061433447098977e-06, "loss": 0.0391, "num_tokens": 136087432.0, "reward": 1.2000000476837158, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 1.718477487564087, "sampling/importance_sampling_ratio/mean": 0.41791361570358276, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9123876094818115, "sampling/sampling_logp_difference/mean": 0.013809124007821083, "step": 1289, "step_time": 73.49020410864614 }, { "clip_ratio/high_max": 0.01311620493652299, "clip_ratio/high_mean": 0.004269160941476002, "clip_ratio/low_mean": 0.0026120699621969834, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006881230976432562, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 777.375, "completions/mean_terminated_length": 443.70587158203125, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.10339153930544853, "epoch": 0.7337883959044369, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02448083832859993, "kl": 0.013241183594800532, "learning_rate": 2.4010238907849827e-06, "loss": 0.0292, "num_tokens": 136197391.0, "reward": 0.7250000238418579, "reward_std": 0.905467689037323, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.8686699867248535, "sampling/importance_sampling_ratio/mean": 0.344041645526886, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.704271078109741, "sampling/sampling_logp_difference/mean": 0.014671295881271362, "step": 1290, "step_time": 74.50825007888488 }, { "clip_ratio/high_max": 0.008802940836176276, "clip_ratio/high_mean": 0.0026001364458352327, "clip_ratio/low_mean": 0.002490234372089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005090370774269104, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 877.4750366210938, "completions/mean_terminated_length": 679.2352905273438, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.11932695377618074, "epoch": 0.7343572241183163, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022159336134791374, "kl": 0.012481850804761052, "learning_rate": 2.3959044368600686e-06, "loss": -0.0066, "num_tokens": 136301466.0, "reward": 0.824999988079071, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.8188902139663696, "sampling/importance_sampling_ratio/mean": 0.37521785497665405, "sampling/importance_sampling_ratio/min": 0.002823447110131383, "sampling/sampling_logp_difference/max": 2.4608287811279297, "sampling/sampling_logp_difference/mean": 0.014062101021409035, "step": 1291, "step_time": 74.00197176798247 }, { "clip_ratio/high_max": 0.014643788803368807, "clip_ratio/high_mean": 0.004534698819043115, "clip_ratio/low_mean": 0.0008869166267686523, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00542161549674347, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 828.2250366210938, "completions/mean_terminated_length": 651.0952758789062, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.11644560005515814, "epoch": 0.7349260523321957, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01609925925731659, "kl": 0.010701107152272016, "learning_rate": 2.3907849829351536e-06, "loss": -0.043, "num_tokens": 136391115.0, "reward": 0.925000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.573291301727295, "sampling/importance_sampling_ratio/mean": 0.3393499553203583, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6990818977355957, "sampling/sampling_logp_difference/mean": 0.014607056975364685, "step": 1292, "step_time": 74.47484818589874 }, { "clip_ratio/high_max": 0.007497376180253923, "clip_ratio/high_mean": 0.0022305411694105715, "clip_ratio/low_mean": 0.002416992181679234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004647533380193636, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 812.0, "completions/mean_length": 955.6500244140625, "completions/mean_terminated_length": 682.25, "completions/min_length": 464.0, "completions/min_terminated_length": 464.0, "entropy": 0.13400138169527054, "epoch": 0.735494880546075, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.06640540808439255, "kl": 0.012584323063492775, "learning_rate": 2.385665529010239e-06, "loss": 0.0655, "num_tokens": 136508589.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.1940793991088867, "sampling/importance_sampling_ratio/mean": 0.3232523500919342, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.703598976135254, "sampling/sampling_logp_difference/mean": 0.014477468095719814, "step": 1293, "step_time": 79.2961608665064 }, { "clip_ratio/high_max": 0.016470429487526417, "clip_ratio/high_mean": 0.006406691973097622, "clip_ratio/low_mean": 0.0029955881473142654, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00940228020772338, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 750.8500366210938, "completions/mean_terminated_length": 659.800048828125, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "entropy": 0.16479326225817204, "epoch": 0.7360637087599545, "frac_reward_zero_std": 0.0, "grad_norm": 0.033989835530519485, "kl": 0.026406970690004528, "learning_rate": 2.380546075085324e-06, "loss": -0.0086, "num_tokens": 136610431.0, "reward": 1.25, "reward_std": 0.8397191166877747, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.0677549839019775, "sampling/importance_sampling_ratio/mean": 0.41344624757766724, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7100050449371338, "sampling/sampling_logp_difference/mean": 0.018385985866189003, "step": 1294, "step_time": 70.41213158727624 }, { "clip_ratio/high_max": 0.012611220357939601, "clip_ratio/high_mean": 0.004530161386355758, "clip_ratio/low_mean": 0.002522428696465795, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007052589993691072, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 877.6749877929688, "completions/mean_terminated_length": 633.800048828125, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.13698745146393776, "epoch": 0.7366325369738339, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.05963483452796936, "kl": 0.013168721343390644, "learning_rate": 2.37542662116041e-06, "loss": -0.0868, "num_tokens": 136719834.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.84255051612854, "sampling/importance_sampling_ratio/mean": 0.3773505687713623, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.67610502243042, "sampling/sampling_logp_difference/mean": 0.015886595472693443, "step": 1295, "step_time": 73.87025288003497 }, { "clip_ratio/high_max": 0.008609056880231947, "clip_ratio/high_mean": 0.002780311508104205, "clip_ratio/low_mean": 0.0006591796882275958, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003439491178141907, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 752.7250366210938, "completions/mean_terminated_length": 606.6538696289062, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.1100413454696536, "epoch": 0.7372013651877133, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022773459553718567, "kl": 0.03389009600505233, "learning_rate": 2.370307167235495e-06, "loss": -0.0381, "num_tokens": 136820999.0, "reward": 1.0500000715255737, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.2213475704193115, "sampling/importance_sampling_ratio/mean": 0.38015109300613403, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7591724395751953, "sampling/sampling_logp_difference/mean": 0.014775681309401989, "step": 1296, "step_time": 73.3175122900866 }, { "clip_ratio/high_max": 0.011575804557651281, "clip_ratio/high_mean": 0.004471031890716404, "clip_ratio/low_mean": 0.0022949218546273187, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.00676595390541479, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 904.0, "completions/mean_length": 834.1000366210938, "completions/mean_terminated_length": 602.0, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.09187442902475595, "epoch": 0.7377701934015927, "frac_reward_zero_std": 0.0, "grad_norm": 0.02435721643269062, "kl": 0.012647060444578528, "learning_rate": 2.3651877133105803e-06, "loss": -0.0261, "num_tokens": 136914467.0, "reward": 0.824999988079071, "reward_std": 0.957761824131012, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.14912748336792, "sampling/importance_sampling_ratio/mean": 0.3714138865470886, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6617162227630615, "sampling/sampling_logp_difference/mean": 0.011667660437524319, "step": 1297, "step_time": 70.11377570196055 }, { "clip_ratio/high_max": 0.0015641292557120323, "clip_ratio/high_mean": 0.0005350480787456036, "clip_ratio/low_mean": 0.0010986328270519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016336809057975188, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 959.0, "completions/mean_length": 702.3250122070312, "completions/mean_terminated_length": 529.1154174804688, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.09075693320482969, "epoch": 0.7383390216154722, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.018179653212428093, "kl": 0.013766169489827007, "learning_rate": 2.3600682593856658e-06, "loss": 0.0555, "num_tokens": 137016552.0, "reward": 1.3000000715255737, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.9190647602081299, "sampling/importance_sampling_ratio/mean": 0.43492692708969116, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.337888717651367, "sampling/sampling_logp_difference/mean": 0.013623612932860851, "step": 1298, "step_time": 70.75089706014842 }, { "clip_ratio/high_max": 0.01768746378365904, "clip_ratio/high_mean": 0.006434258655644953, "clip_ratio/low_mean": 0.0026855468895519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009119805647060275, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 830.75, "completions/mean_terminated_length": 637.5, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.1427106074988842, "epoch": 0.7389078498293515, "frac_reward_zero_std": 0.0, "grad_norm": 0.039085753262043, "kl": 0.01367460680194199, "learning_rate": 2.354948805460751e-06, "loss": -0.1301, "num_tokens": 137125726.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.0095345973968506, "sampling/importance_sampling_ratio/mean": 0.3576175272464752, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5893430709838867, "sampling/sampling_logp_difference/mean": 0.01615484245121479, "step": 1299, "step_time": 73.92378459055908 }, { "clip_ratio/high_max": 0.014778449200093746, "clip_ratio/high_mean": 0.004354681557742879, "clip_ratio/low_mean": 0.0018066406082652975, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0061613223515450954, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 887.6500244140625, "completions/mean_terminated_length": 683.125, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.0956551730632782, "epoch": 0.7394766780432309, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.010013164021074772, "kl": 0.01886417338391766, "learning_rate": 2.3498293515358362e-06, "loss": 0.0124, "num_tokens": 137227720.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.3621547222137451, "sampling/importance_sampling_ratio/mean": 0.2256462275981903, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.8786165714263916, "sampling/sampling_logp_difference/mean": 0.012080180458724499, "step": 1300, "step_time": 76.2089903508313 }, { "clip_ratio/high_max": 0.011029614368453622, "clip_ratio/high_mean": 0.002833736565662548, "clip_ratio/low_mean": 0.0011430220292822924, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003976758598582819, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 804.3500366210938, "completions/mean_terminated_length": 584.7000122070312, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.1053757993504405, "epoch": 0.7400455062571104, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.028772400692105293, "kl": 0.011817366466857493, "learning_rate": 2.3447098976109216e-06, "loss": -0.0699, "num_tokens": 137336630.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.719268798828125, "sampling/importance_sampling_ratio/mean": 0.47431594133377075, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9509010314941406, "sampling/sampling_logp_difference/mean": 0.012617933563888073, "step": 1301, "step_time": 75.28036989318207 }, { "clip_ratio/high_max": 0.009941681171767414, "clip_ratio/high_mean": 0.0027222510980209336, "clip_ratio/low_mean": 0.0007475754428014625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003469826508080587, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 785.2000122070312, "completions/mean_terminated_length": 670.2222290039062, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.13869775086641312, "epoch": 0.7406143344709898, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.016155775636434555, "kl": 0.01316721155308187, "learning_rate": 2.339590443686007e-06, "loss": -0.0675, "num_tokens": 137435206.0, "reward": 1.225000023841858, "reward_std": 0.9195176959037781, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.072141170501709, "sampling/importance_sampling_ratio/mean": 0.37537044286727905, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.2490386962890625, "sampling/sampling_logp_difference/mean": 0.015760943293571472, "step": 1302, "step_time": 77.12413252424449 }, { "clip_ratio/high_max": 0.01692154409829527, "clip_ratio/high_mean": 0.004994940623873845, "clip_ratio/low_mean": 0.002050781244179234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0070457219844684005, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 949.9249877929688, "completions/mean_terminated_length": 777.0833740234375, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "entropy": 0.16554183326661587, "epoch": 0.7411831626848692, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02891288325190544, "kl": 0.010981336527038366, "learning_rate": 2.334470989761092e-06, "loss": -0.0355, "num_tokens": 137549899.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.7944371700286865, "sampling/importance_sampling_ratio/mean": 0.2930501699447632, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6987969875335693, "sampling/sampling_logp_difference/mean": 0.01651710644364357, "step": 1303, "step_time": 77.73741369298659 }, { "clip_ratio/high_max": 0.013272305252030492, "clip_ratio/high_mean": 0.003773016927880235, "clip_ratio/low_mean": 0.0013427734375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005115790409035981, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 903.2250366210938, "completions/mean_terminated_length": 652.3846435546875, "completions/min_length": 406.0, "completions/min_terminated_length": 406.0, "entropy": 0.08726969547569752, "epoch": 0.7417519908987485, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.006696473341435194, "kl": 0.010265288932714611, "learning_rate": 2.3293515358361775e-06, "loss": 0.0146, "num_tokens": 137665580.0, "reward": 0.6000000238418579, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.33663809299469, "sampling/importance_sampling_ratio/mean": 0.22985315322875977, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.072493553161621, "sampling/sampling_logp_difference/mean": 0.011488696560263634, "step": 1304, "step_time": 75.82623640401289 }, { "clip_ratio/high_max": 0.013344980776309967, "clip_ratio/high_mean": 0.0031122266518650576, "clip_ratio/low_mean": 0.0022705078081344254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00538273440906778, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 885.0, "completions/mean_length": 907.875, "completions/mean_terminated_length": 601.727294921875, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.08286766521632671, "epoch": 0.742320819112628, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02627863734960556, "kl": 0.008040405053179711, "learning_rate": 2.3242320819112625e-06, "loss": 0.0073, "num_tokens": 137770887.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.4480044841766357, "sampling/importance_sampling_ratio/mean": 0.5323532223701477, "sampling/importance_sampling_ratio/min": 0.00043959313188679516, "sampling/sampling_logp_difference/max": 1.5284864902496338, "sampling/sampling_logp_difference/mean": 0.01036444865167141, "step": 1305, "step_time": 73.73003646102734 }, { "clip_ratio/high_max": 0.008247749414294958, "clip_ratio/high_mean": 0.002644931839313358, "clip_ratio/low_mean": 0.002587890630820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005232822411926463, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 894.9000244140625, "completions/mean_terminated_length": 701.25, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.2209399975836277, "epoch": 0.7428896473265074, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01264040544629097, "kl": 0.01593101571779698, "learning_rate": 2.3191126279863484e-06, "loss": -0.0045, "num_tokens": 137884411.0, "reward": 0.625, "reward_std": 0.8378084897994995, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.9229907989501953, "sampling/importance_sampling_ratio/mean": 0.23884062469005585, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8639070987701416, "sampling/sampling_logp_difference/mean": 0.020976591855287552, "step": 1306, "step_time": 76.46669914503582 }, { "clip_ratio/high_max": 0.01629864051938057, "clip_ratio/high_mean": 0.0067778395605273545, "clip_ratio/low_mean": 0.0015592246636515483, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008337064005900174, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 763.1000366210938, "completions/mean_terminated_length": 589.1666870117188, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.11806148570030928, "epoch": 0.7434584755403868, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.05061811953783035, "kl": 0.014095218270085752, "learning_rate": 2.3139931740614334e-06, "loss": -0.0142, "num_tokens": 137984967.0, "reward": 1.100000023841858, "reward_std": 0.9554163813591003, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.888031482696533, "sampling/importance_sampling_ratio/mean": 0.38022103905677795, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2785305976867676, "sampling/sampling_logp_difference/mean": 0.015992451459169388, "step": 1307, "step_time": 72.17946240422316 }, { "clip_ratio/high_max": 0.012426840025000274, "clip_ratio/high_mean": 0.0038274359103525057, "clip_ratio/low_mean": 0.0020996093371650204, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005927045160206035, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 919.2750244140625, "completions/mean_terminated_length": 701.769287109375, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.08639733958989382, "epoch": 0.7440273037542662, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03616740182042122, "kl": 0.011503821704536676, "learning_rate": 2.308873720136519e-06, "loss": 0.0584, "num_tokens": 138089562.0, "reward": 0.574999988079071, "reward_std": 0.873762845993042, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.9292092323303223, "sampling/importance_sampling_ratio/mean": 0.611904501914978, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.439821481704712, "sampling/sampling_logp_difference/mean": 0.01103143859654665, "step": 1308, "step_time": 74.20264878077433 }, { "clip_ratio/high_max": 0.008074840356130153, "clip_ratio/high_mean": 0.002167611244658474, "clip_ratio/low_mean": 0.0033935547253349796, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00556116591906175, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 927.0, "completions/mean_length": 806.9249877929688, "completions/mean_terminated_length": 541.6111450195312, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.13108259160071611, "epoch": 0.7445961319681457, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03210393711924553, "kl": 0.014350459794513881, "learning_rate": 2.303754266211604e-06, "loss": 0.0389, "num_tokens": 138200575.0, "reward": 0.8500000238418579, "reward_std": 0.9753369092941284, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.9709372520446777, "sampling/importance_sampling_ratio/mean": 0.37052255868911743, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8378877639770508, "sampling/sampling_logp_difference/mean": 0.016122838482260704, "step": 1309, "step_time": 82.46394015219994 }, { "clip_ratio/high_max": 0.014264128520153463, "clip_ratio/high_mean": 0.004874319361988455, "clip_ratio/low_mean": 0.0026569001347525045, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0075312197441235185, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 850.4249877929688, "completions/mean_terminated_length": 638.2777709960938, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.14197090175002813, "epoch": 0.745164960182025, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.016611332073807716, "kl": 0.013370672590099275, "learning_rate": 2.2986348122866897e-06, "loss": -0.0397, "num_tokens": 138312096.0, "reward": 0.824999988079071, "reward_std": 0.957761824131012, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.7158057689666748, "sampling/importance_sampling_ratio/mean": 0.22045548260211945, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0557456016540527, "sampling/sampling_logp_difference/mean": 0.016395097598433495, "step": 1310, "step_time": 76.30567630194128 }, { "clip_ratio/high_max": 0.010489230044186115, "clip_ratio/high_mean": 0.002329327500774525, "clip_ratio/low_mean": 0.0009521484535071068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032814759761095047, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 938.0, "completions/mean_length": 868.8250122070312, "completions/mean_terminated_length": 636.0625, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.10885570105165243, "epoch": 0.7457337883959044, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.031010370701551437, "kl": 0.009160713176243007, "learning_rate": 2.2935153583617747e-06, "loss": -0.0444, "num_tokens": 138418953.0, "reward": 0.625, "reward_std": 0.8378084897994995, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 1.7724517583847046, "sampling/importance_sampling_ratio/mean": 0.4087265133857727, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.313706398010254, "sampling/sampling_logp_difference/mean": 0.012927213683724403, "step": 1311, "step_time": 73.67487378977239 }, { "clip_ratio/high_max": 0.01362149603664875, "clip_ratio/high_mean": 0.004562202811939642, "clip_ratio/low_mean": 0.002880859377910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007443062146194279, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 776.0, "completions/mean_length": 870.3500366210938, "completions/mean_terminated_length": 511.8333435058594, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.14758231677114964, "epoch": 0.7463026166097838, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01976191997528076, "kl": 0.01642580411862582, "learning_rate": 2.28839590443686e-06, "loss": 0.0634, "num_tokens": 138519975.0, "reward": 0.550000011920929, "reward_std": 0.875594973564148, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.6103689670562744, "sampling/importance_sampling_ratio/mean": 0.27884215116500854, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.023287773132324, "sampling/sampling_logp_difference/mean": 0.01610632799565792, "step": 1312, "step_time": 71.64508272637613 }, { "clip_ratio/high_max": 0.007516500831115991, "clip_ratio/high_mean": 0.0020519182580756024, "clip_ratio/low_mean": 0.0017089843895519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037609025894198567, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 964.7999877929688, "completions/mean_terminated_length": 728.0, "completions/min_length": 435.0, "completions/min_terminated_length": 435.0, "entropy": 0.10553454328328371, "epoch": 0.7468714448236633, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024657493457198143, "kl": 0.009587966720573604, "learning_rate": 2.283276450511945e-06, "loss": -0.087, "num_tokens": 138625799.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.82511568069458, "sampling/importance_sampling_ratio/mean": 0.4426063895225525, "sampling/importance_sampling_ratio/min": 0.0038492667954415083, "sampling/sampling_logp_difference/max": 2.104776620864868, "sampling/sampling_logp_difference/mean": 0.011894459836184978, "step": 1313, "step_time": 74.71205752785318 }, { "clip_ratio/high_max": 0.006634265941102058, "clip_ratio/high_mean": 0.0017451240419177338, "clip_ratio/low_mean": 0.002832031241268851, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004577155283186585, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 879.5250244140625, "completions/mean_terminated_length": 611.2142944335938, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.12718109413981438, "epoch": 0.7474402730375427, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0419946052134037, "kl": 0.010679712169803679, "learning_rate": 2.278156996587031e-06, "loss": -0.0524, "num_tokens": 138735980.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.6357815265655518, "sampling/importance_sampling_ratio/mean": 0.5889689326286316, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.053593635559082, "sampling/sampling_logp_difference/mean": 0.014358914457261562, "step": 1314, "step_time": 75.30606260849163 }, { "clip_ratio/high_max": 0.012621485511772335, "clip_ratio/high_mean": 0.004540528170764446, "clip_ratio/low_mean": 0.002975439580040984, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007515967707149684, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 941.0, "completions/mean_length": 766.6749877929688, "completions/mean_terminated_length": 576.478271484375, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.10161834303289652, "epoch": 0.7480091012514221, "frac_reward_zero_std": 0.0, "grad_norm": 0.0964517891407013, "kl": 0.01052279822761193, "learning_rate": 2.273037542662116e-06, "loss": -0.1586, "num_tokens": 138835751.0, "reward": 1.0500000715255737, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.9073967933654785, "sampling/importance_sampling_ratio/mean": 0.48947858810424805, "sampling/importance_sampling_ratio/min": 0.003185276873409748, "sampling/sampling_logp_difference/max": 1.4521963596343994, "sampling/sampling_logp_difference/mean": 0.012993819080293179, "step": 1315, "step_time": 70.4274073606357 }, { "clip_ratio/high_max": 0.008771934342803434, "clip_ratio/high_mean": 0.0021426444509415887, "clip_ratio/low_mean": 0.0006103515734139364, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002752996006165631, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 731.6000366210938, "completions/mean_terminated_length": 536.6666870117188, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.08913659676909447, "epoch": 0.7485779294653014, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011251860298216343, "kl": 0.015323525411076844, "learning_rate": 2.2679180887372015e-06, "loss": 0.0754, "num_tokens": 138941007.0, "reward": 1.0500000715255737, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.661508321762085, "sampling/importance_sampling_ratio/mean": 0.37520861625671387, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8813939094543457, "sampling/sampling_logp_difference/mean": 0.01422024518251419, "step": 1316, "step_time": 76.29479447891936 }, { "clip_ratio/high_max": 0.018055097898468375, "clip_ratio/high_mean": 0.008445637999102473, "clip_ratio/low_mean": 0.0016357421827706276, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010081380256451666, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 889.0, "completions/mean_length": 750.7250366210938, "completions/mean_terminated_length": 568.5416870117188, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.1265092734247446, "epoch": 0.7491467576791809, "frac_reward_zero_std": 0.0, "grad_norm": 0.022381410002708435, "kl": 0.014811878907494247, "learning_rate": 2.2627986348122865e-06, "loss": 0.0154, "num_tokens": 139045212.0, "reward": 1.2000000476837158, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.4220871925354004, "sampling/importance_sampling_ratio/mean": 0.34029635787010193, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.311746597290039, "sampling/sampling_logp_difference/mean": 0.015328237786889076, "step": 1317, "step_time": 74.60809803404845 }, { "clip_ratio/high_max": 0.008460273005766794, "clip_ratio/high_mean": 0.002592581047792919, "clip_ratio/low_mean": 0.0014648437791038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004057424870552495, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 898.0, "completions/mean_length": 786.125, "completions/mean_terminated_length": 495.3888854980469, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.0856280354782939, "epoch": 0.7497155858930603, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014665130525827408, "kl": 0.013538626837544143, "learning_rate": 2.2576791808873723e-06, "loss": 0.0967, "num_tokens": 139153361.0, "reward": 0.75, "reward_std": 0.8987169861793518, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.8226773738861084, "sampling/importance_sampling_ratio/mean": 0.5696627497673035, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6007556915283203, "sampling/sampling_logp_difference/mean": 0.012237928807735443, "step": 1318, "step_time": 72.25111986882985 }, { "clip_ratio/high_max": 0.014131775475107133, "clip_ratio/high_mean": 0.005854562972672284, "clip_ratio/low_mean": 0.002246093761641532, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.008100656676106155, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 799.1000366210938, "completions/mean_terminated_length": 574.2000122070312, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.11302639171481133, "epoch": 0.7502844141069397, "frac_reward_zero_std": 0.0, "grad_norm": 0.018171288073062897, "kl": 0.018115393701009452, "learning_rate": 2.2525597269624573e-06, "loss": 0.0339, "num_tokens": 139252125.0, "reward": 0.875, "reward_std": 0.9388344883918762, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.567739486694336, "sampling/importance_sampling_ratio/mean": 0.3385724425315857, "sampling/importance_sampling_ratio/min": 0.0012610304402187467, "sampling/sampling_logp_difference/max": 3.8633084297180176, "sampling/sampling_logp_difference/mean": 0.013950004242360592, "step": 1319, "step_time": 74.44629630516283 }, { "clip_ratio/high_max": 0.009808513103052974, "clip_ratio/high_mean": 0.0035963390837423503, "clip_ratio/low_mean": 0.0013755005238635931, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004971839574864134, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 832.0499877929688, "completions/mean_terminated_length": 640.1000366210938, "completions/min_length": 420.0, "completions/min_terminated_length": 420.0, "entropy": 0.10263306275010109, "epoch": 0.7508532423208191, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03556206822395325, "kl": 0.012984139961190522, "learning_rate": 2.2474402730375428e-06, "loss": 0.0783, "num_tokens": 139360775.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.6948184967041016, "sampling/importance_sampling_ratio/mean": 0.4313167631626129, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.608675241470337, "sampling/sampling_logp_difference/mean": 0.013943994417786598, "step": 1320, "step_time": 73.06273916386999 }, { "clip_ratio/high_max": 0.009120563976466656, "clip_ratio/high_mean": 0.0024679438647581264, "clip_ratio/low_mean": 0.0027099609142169356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005177904822630808, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 948.5499877929688, "completions/mean_terminated_length": 822.800048828125, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "entropy": 0.10067976079881191, "epoch": 0.7514220705346986, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023340953513979912, "kl": 0.012540742114651948, "learning_rate": 2.2423208191126278e-06, "loss": 0.0126, "num_tokens": 139461597.0, "reward": 0.675000011920929, "reward_std": 0.9167249202728271, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.7046444416046143, "sampling/importance_sampling_ratio/mean": 0.5285469889640808, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6363744735717773, "sampling/sampling_logp_difference/mean": 0.012614025734364986, "step": 1321, "step_time": 73.11199871473946 }, { "clip_ratio/high_max": 0.013716657762415707, "clip_ratio/high_mean": 0.0032630276982672513, "clip_ratio/low_mean": 0.0017089843822759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004972012146026827, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 802.0, "completions/mean_length": 899.8500366210938, "completions/mean_terminated_length": 472.22222900390625, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.06772800418548286, "epoch": 0.7519908987485779, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022641021758317947, "kl": 0.011190462450031191, "learning_rate": 2.2372013651877132e-06, "loss": -0.0101, "num_tokens": 139564591.0, "reward": 0.45000001788139343, "reward_std": 0.8458041548728943, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.7215185165405273, "sampling/importance_sampling_ratio/mean": 0.5116069316864014, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1187782287597656, "sampling/sampling_logp_difference/mean": 0.009451723657548428, "step": 1322, "step_time": 72.37349881930277 }, { "clip_ratio/high_max": 0.012579339789226651, "clip_ratio/high_mean": 0.0028962276119273156, "clip_ratio/low_mean": 0.001043612102876068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0039398396329488605, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 932.0, "completions/mean_length": 954.875, "completions/mean_terminated_length": 716.7777709960938, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.09929254464805126, "epoch": 0.7525597269624573, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.0040532369166612625, "kl": 0.009253317839466035, "learning_rate": 2.2320819112627987e-06, "loss": -0.0026, "num_tokens": 139681138.0, "reward": 0.3499999940395355, "reward_std": 0.6998168230056763, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 1.1994943618774414, "sampling/importance_sampling_ratio/mean": 0.23132558166980743, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7528369426727295, "sampling/sampling_logp_difference/mean": 0.012160345911979675, "step": 1323, "step_time": 79.51400464680046 }, { "clip_ratio/high_max": 0.009378439746797085, "clip_ratio/high_mean": 0.002162983982998412, "clip_ratio/low_mean": 0.0034912109840661287, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005654194857925177, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 883.0, "completions/mean_length": 955.3250122070312, "completions/mean_terminated_length": 680.625, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.18444395251572132, "epoch": 0.7531285551763367, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.017412077635526657, "kl": 0.015060942620038986, "learning_rate": 2.226962457337884e-06, "loss": -0.0215, "num_tokens": 139799719.0, "reward": 0.3499999940395355, "reward_std": 0.7355444431304932, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 1.8000261783599854, "sampling/importance_sampling_ratio/mean": 0.2593168318271637, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5262699127197266, "sampling/sampling_logp_difference/mean": 0.017561644315719604, "step": 1324, "step_time": 77.41233389428817 }, { "clip_ratio/high_max": 0.006822089897468686, "clip_ratio/high_mean": 0.001961269634193741, "clip_ratio/low_mean": 0.001092810278350953, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003054079890716821, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 956.0, "completions/mean_length": 937.9750366210938, "completions/mean_terminated_length": 778.2142944335938, "completions/min_length": 526.0, "completions/min_terminated_length": 526.0, "entropy": 0.06926998170092702, "epoch": 0.7536973833902162, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025645464658737183, "kl": 0.007630132720805705, "learning_rate": 2.221843003412969e-06, "loss": -0.0525, "num_tokens": 139885326.0, "reward": 0.6000000238418579, "reward_std": 0.8711913824081421, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.0957446098327637, "sampling/importance_sampling_ratio/mean": 0.5102130174636841, "sampling/importance_sampling_ratio/min": 0.015521076507866383, "sampling/sampling_logp_difference/max": 2.232272148132324, "sampling/sampling_logp_difference/mean": 0.009305504150688648, "step": 1325, "step_time": 67.93553994852118 }, { "clip_ratio/high_max": 0.014567179779987782, "clip_ratio/high_mean": 0.005104984244098887, "clip_ratio/low_mean": 0.0014648437645519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006569828081410378, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 898.0, "completions/mean_length": 707.2000122070312, "completions/mean_terminated_length": 536.6154174804688, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.09493114240467548, "epoch": 0.7542662116040956, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01706419698894024, "kl": 0.016162230051122606, "learning_rate": 2.2167235494880545e-06, "loss": -0.0163, "num_tokens": 139989550.0, "reward": 1.3000000715255737, "reward_std": 0.9660917520523071, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.359837293624878, "sampling/importance_sampling_ratio/mean": 0.4048793911933899, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.073554515838623, "sampling/sampling_logp_difference/mean": 0.014386243186891079, "step": 1326, "step_time": 73.49930618004873 }, { "clip_ratio/high_max": 0.012685946305282414, "clip_ratio/high_mean": 0.004699549201177433, "clip_ratio/low_mean": 0.002001953136641532, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006701502250507474, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 901.7999877929688, "completions/mean_terminated_length": 698.1333618164062, "completions/min_length": 453.0, "completions/min_terminated_length": 453.0, "entropy": 0.10868613515049219, "epoch": 0.754835039817975, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.025385765358805656, "kl": 0.009609711705707014, "learning_rate": 2.21160409556314e-06, "loss": 0.017, "num_tokens": 140102446.0, "reward": 0.574999988079071, "reward_std": 0.8129560351371765, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.3443970680236816, "sampling/importance_sampling_ratio/mean": 0.5347447395324707, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7529175281524658, "sampling/sampling_logp_difference/mean": 0.013029825873672962, "step": 1327, "step_time": 76.21230606525205 }, { "clip_ratio/high_max": 0.008482845441903919, "clip_ratio/high_mean": 0.002062525018118322, "clip_ratio/low_mean": 0.0026855468822759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004748071834910661, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 881.0, "completions/mean_length": 949.3500366210938, "completions/mean_terminated_length": 597.4285888671875, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "entropy": 0.09835527837276459, "epoch": 0.7554038680318543, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027438392862677574, "kl": 0.012137910351157188, "learning_rate": 2.2064846416382254e-06, "loss": 0.04, "num_tokens": 140218260.0, "reward": 0.30000001192092896, "reward_std": 0.6868732571601868, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.4117796421051025, "sampling/importance_sampling_ratio/mean": 0.3926827609539032, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.80963397026062, "sampling/sampling_logp_difference/mean": 0.01095602847635746, "step": 1328, "step_time": 77.09090826497413 }, { "clip_ratio/high_max": 0.008108649693895131, "clip_ratio/high_mean": 0.002577244085841812, "clip_ratio/low_mean": 0.0019775390555878403, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004554783052299172, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 935.875, "completions/mean_terminated_length": 730.25, "completions/min_length": 513.0, "completions/min_terminated_length": 513.0, "entropy": 0.09078668057918549, "epoch": 0.7559726962457338, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.028536934405565262, "kl": 0.011182154004927725, "learning_rate": 2.2013651877133104e-06, "loss": 0.1029, "num_tokens": 140330575.0, "reward": 0.4000000059604645, "reward_std": 0.6717752814292908, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.357459783554077, "sampling/importance_sampling_ratio/mean": 0.31305328011512756, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4554905891418457, "sampling/sampling_logp_difference/mean": 0.01152876764535904, "step": 1329, "step_time": 77.45997773762792 }, { "clip_ratio/high_max": 0.014283394208177924, "clip_ratio/high_mean": 0.004798989510163665, "clip_ratio/low_mean": 0.0014892578074068297, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062882472411729395, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 770.2250366210938, "completions/mean_terminated_length": 661.4642944335938, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "entropy": 0.13129760045558214, "epoch": 0.7565415244596132, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03729915991425514, "kl": 0.014123418601229787, "learning_rate": 2.196245733788396e-06, "loss": 0.0233, "num_tokens": 140439840.0, "reward": 1.350000023841858, "reward_std": 0.9212588667869568, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.827732801437378, "sampling/importance_sampling_ratio/mean": 0.506028950214386, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4898533821105957, "sampling/sampling_logp_difference/mean": 0.01611064188182354, "step": 1330, "step_time": 76.06744122901 }, { "clip_ratio/high_max": 0.011506712180562317, "clip_ratio/high_mean": 0.004443750647624256, "clip_ratio/low_mean": 0.0020751953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006518945941934362, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 953.0, "completions/mean_length": 846.2750244140625, "completions/mean_terminated_length": 579.6875, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.15439517423510551, "epoch": 0.7571103526734926, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.038004133850336075, "kl": 0.019860255415551364, "learning_rate": 2.1911262798634813e-06, "loss": -0.2045, "num_tokens": 140551323.0, "reward": 0.75, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.6523263454437256, "sampling/importance_sampling_ratio/mean": 0.4929896295070648, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.562748432159424, "sampling/sampling_logp_difference/mean": 0.01657094620168209, "step": 1331, "step_time": 75.25049722054973 }, { "clip_ratio/high_max": 0.00987282715504989, "clip_ratio/high_mean": 0.0020027185819344595, "clip_ratio/low_mean": 0.0010009765574068297, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030036950920475647, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 734.6500244140625, "completions/mean_terminated_length": 520.7825927734375, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.08719322551041842, "epoch": 0.757679180887372, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 6.170569896697998, "kl": 0.7932859383290634, "learning_rate": 2.1860068259385667e-06, "loss": 0.1413, "num_tokens": 140663781.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.806673049926758, "sampling/importance_sampling_ratio/mean": 0.47780314087867737, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.540475606918335, "sampling/sampling_logp_difference/mean": 0.013275078497827053, "step": 1332, "step_time": 73.28753715357743 }, { "clip_ratio/high_max": 0.014111490454524755, "clip_ratio/high_mean": 0.006405584441381507, "clip_ratio/low_mean": 0.0011474609636934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007553045637905598, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 805.4750366210938, "completions/mean_terminated_length": 674.3599853515625, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.10160529147833586, "epoch": 0.7582480091012515, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024185623973608017, "kl": 0.010366045462433249, "learning_rate": 2.1808873720136517e-06, "loss": -0.0406, "num_tokens": 140764088.0, "reward": 1.125, "reward_std": 0.9388344883918762, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.767972707748413, "sampling/importance_sampling_ratio/mean": 0.5969168543815613, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1972193717956543, "sampling/sampling_logp_difference/mean": 0.013389802537858486, "step": 1333, "step_time": 74.01219583908096 }, { "clip_ratio/high_max": 0.007552429102361202, "clip_ratio/high_mean": 0.0015104858321137726, "clip_ratio/low_mean": 0.0010742187678260962, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0025847045762930065, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 729.125, "completions/mean_terminated_length": 434.25, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.07984318863600492, "epoch": 0.7588168373151308, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009205833077430725, "kl": 0.009372433996759355, "learning_rate": 2.175767918088737e-06, "loss": 0.0951, "num_tokens": 140868621.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.8812761306762695, "sampling/importance_sampling_ratio/mean": 0.3083323836326599, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1508634090423584, "sampling/sampling_logp_difference/mean": 0.01134351547807455, "step": 1334, "step_time": 69.6322560987901 }, { "clip_ratio/high_max": 0.011463836766779423, "clip_ratio/high_mean": 0.0042889614996965975, "clip_ratio/low_mean": 0.0022705078299622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006559469096828252, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 968.0, "completions/mean_length": 918.0250244140625, "completions/mean_terminated_length": 788.5, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.1374314036220312, "epoch": 0.7593856655290102, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015110976062715054, "kl": 0.011841936502605677, "learning_rate": 2.1706484641638226e-06, "loss": 0.0355, "num_tokens": 140975326.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.4523581266403198, "sampling/importance_sampling_ratio/mean": 0.3237127661705017, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.104957103729248, "sampling/sampling_logp_difference/mean": 0.014897379092872143, "step": 1335, "step_time": 81.81427975534461 }, { "clip_ratio/high_max": 0.017535825143568218, "clip_ratio/high_mean": 0.007820737781003118, "clip_ratio/low_mean": 0.0014763799590582494, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009297117707319558, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 660.7000122070312, "completions/mean_terminated_length": 555.2257690429688, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.10311801079660654, "epoch": 0.7599544937428896, "frac_reward_zero_std": 0.0, "grad_norm": 0.042484890669584274, "kl": 0.01714252564124763, "learning_rate": 2.165529010238908e-06, "loss": 0.0522, "num_tokens": 141079730.0, "reward": 1.4500000476837158, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.8939807415008545, "sampling/importance_sampling_ratio/mean": 0.4711896479129791, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1372358798980713, "sampling/sampling_logp_difference/mean": 0.01439211517572403, "step": 1336, "step_time": 77.21318179415539 }, { "clip_ratio/high_max": 0.010098297148942947, "clip_ratio/high_mean": 0.0035086415737168863, "clip_ratio/low_mean": 0.0014648437390860636, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004973485192749649, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 825.4500122070312, "completions/mean_terminated_length": 663.0, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "entropy": 0.0943562900647521, "epoch": 0.7605233219567691, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019157761707901955, "kl": 0.014788987929932773, "learning_rate": 2.160409556313993e-06, "loss": 0.0494, "num_tokens": 141178836.0, "reward": 1.100000023841858, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.5405399799346924, "sampling/importance_sampling_ratio/mean": 0.43849724531173706, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.444138526916504, "sampling/sampling_logp_difference/mean": 0.011840703897178173, "step": 1337, "step_time": 73.6429635854438 }, { "clip_ratio/high_max": 0.014468986832071096, "clip_ratio/high_mean": 0.004397370750666596, "clip_ratio/low_mean": 0.0014281743715400808, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0058255452022422105, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 952.0, "completions/mean_length": 646.1749877929688, "completions/mean_terminated_length": 502.862060546875, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.11852008756250143, "epoch": 0.7610921501706485, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02579341270029545, "kl": 0.018365694675594568, "learning_rate": 2.1552901023890785e-06, "loss": -0.0057, "num_tokens": 141270539.0, "reward": 1.225000023841858, "reward_std": 0.8619447350502014, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.723031759262085, "sampling/importance_sampling_ratio/mean": 0.5714975595474243, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.234870672225952, "sampling/sampling_logp_difference/mean": 0.01580187678337097, "step": 1338, "step_time": 66.77037474582903 }, { "clip_ratio/high_max": 0.015595993580063805, "clip_ratio/high_mean": 0.006217211244802456, "clip_ratio/low_mean": 0.002197265606810106, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008414476818870753, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 790.4249877929688, "completions/mean_terminated_length": 556.8500366210938, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.10467015579342842, "epoch": 0.7616609783845278, "frac_reward_zero_std": 0.0, "grad_norm": 0.02285582572221756, "kl": 0.014898298890329897, "learning_rate": 2.150170648464164e-06, "loss": -0.0071, "num_tokens": 141375252.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.470419406890869, "sampling/importance_sampling_ratio/mean": 0.3320496678352356, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.316201686859131, "sampling/sampling_logp_difference/mean": 0.014452241361141205, "step": 1339, "step_time": 73.44589437753893 }, { "clip_ratio/high_max": 0.01420894218608737, "clip_ratio/high_mean": 0.006246023345738649, "clip_ratio/low_mean": 0.0008056640872382559, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007051687396597117, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 970.0, "completions/mean_length": 707.3250122070312, "completions/mean_terminated_length": 628.15625, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "entropy": 0.09564602840691805, "epoch": 0.7622298065984073, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023745235055685043, "kl": 0.015838429797440767, "learning_rate": 2.1450511945392493e-06, "loss": -0.0592, "num_tokens": 141467105.0, "reward": 1.5750000476837158, "reward_std": 0.8129560351371765, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.7749999761581421, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.611907720565796, "sampling/importance_sampling_ratio/mean": 0.2853612005710602, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.384115219116211, "sampling/sampling_logp_difference/mean": 0.013604472391307354, "step": 1340, "step_time": 72.95853597996756 }, { "clip_ratio/high_max": 0.01014091377146542, "clip_ratio/high_mean": 0.002999726348207332, "clip_ratio/low_mean": 0.0024414062354480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005441132641863078, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 857.2250366210938, "completions/mean_terminated_length": 653.388916015625, "completions/min_length": 386.0, "completions/min_terminated_length": 386.0, "entropy": 0.1338935438543558, "epoch": 0.7627986348122867, "frac_reward_zero_std": 0.0, "grad_norm": 0.016660114750266075, "kl": 0.014952168567106128, "learning_rate": 2.1399317406143343e-06, "loss": 0.0111, "num_tokens": 141577474.0, "reward": 0.7250000238418579, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.7824382781982422, "sampling/importance_sampling_ratio/mean": 0.2530616819858551, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.314793586730957, "sampling/sampling_logp_difference/mean": 0.014987215399742126, "step": 1341, "step_time": 79.90937246382236 }, { "clip_ratio/high_max": 0.013092102366499603, "clip_ratio/high_mean": 0.004251840495271608, "clip_ratio/low_mean": 0.0022208257141755894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006472666282206774, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 836.125, "completions/mean_terminated_length": 648.25, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.11555066891014576, "epoch": 0.7633674630261661, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014254257082939148, "kl": 0.012698325677774847, "learning_rate": 2.1348122866894198e-06, "loss": -0.0481, "num_tokens": 141673071.0, "reward": 0.875, "reward_std": 0.9388344883918762, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.6954941749572754, "sampling/importance_sampling_ratio/mean": 0.34375596046447754, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.316737651824951, "sampling/sampling_logp_difference/mean": 0.013424897566437721, "step": 1342, "step_time": 69.79867077316158 }, { "clip_ratio/high_max": 0.008077406615484506, "clip_ratio/high_mean": 0.0017748861573636532, "clip_ratio/low_mean": 0.0021240234236756805, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003898909635609016, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 941.0, "completions/mean_length": 982.7250366210938, "completions/mean_terminated_length": 748.8333740234375, "completions/min_length": 518.0, "completions/min_terminated_length": 518.0, "entropy": 0.0861848397180438, "epoch": 0.7639362912400455, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013102684170007706, "kl": 0.008203960314858705, "learning_rate": 2.1296928327645052e-06, "loss": -0.0419, "num_tokens": 141779196.0, "reward": 0.2750000059604645, "reward_std": 0.6788942813873291, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 1.2244147062301636, "sampling/importance_sampling_ratio/mean": 0.26523035764694214, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1045541763305664, "sampling/sampling_logp_difference/mean": 0.011369292624294758, "step": 1343, "step_time": 73.3014147945214 }, { "clip_ratio/high_max": 0.014945180853828788, "clip_ratio/high_mean": 0.00475683412514627, "clip_ratio/low_mean": 0.00260360496758949, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007360439223702997, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 870.1500244140625, "completions/mean_terminated_length": 700.1052856445312, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.11659697350114584, "epoch": 0.764505119453925, "frac_reward_zero_std": 0.0, "grad_norm": 0.03629685193300247, "kl": 0.012150888389442116, "learning_rate": 2.1245733788395906e-06, "loss": -0.0157, "num_tokens": 141883746.0, "reward": 0.9000000357627869, "reward_std": 0.9818872809410095, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.2677810192108154, "sampling/importance_sampling_ratio/mean": 0.3315860331058502, "sampling/importance_sampling_ratio/min": 0.0005948256584815681, "sampling/sampling_logp_difference/max": 1.6046857833862305, "sampling/sampling_logp_difference/mean": 0.013297063298523426, "step": 1344, "step_time": 76.88933751708828 }, { "clip_ratio/high_max": 0.00734747204114683, "clip_ratio/high_mean": 0.0014694944038637914, "clip_ratio/low_mean": 0.001416015627910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028855100681539625, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 954.5499877929688, "completions/mean_terminated_length": 676.75, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.1215007584542036, "epoch": 0.7650739476678043, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.028324974700808525, "kl": 0.010928306845016778, "learning_rate": 2.119453924914676e-06, "loss": 0.0827, "num_tokens": 141988824.0, "reward": 0.4000000059604645, "reward_std": 0.8101914525032043, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.9777164459228516, "sampling/importance_sampling_ratio/mean": 0.6207228899002075, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.97560453414917, "sampling/sampling_logp_difference/mean": 0.012875537388026714, "step": 1345, "step_time": 72.97438889904879 }, { "clip_ratio/high_max": 0.0031480503093916923, "clip_ratio/high_mean": 0.0006296100546023808, "clip_ratio/low_mean": 0.0003906250040017767, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010202350586041575, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 923.7999877929688, "completions/mean_terminated_length": 659.6364135742188, "completions/min_length": 421.0, "completions/min_terminated_length": 421.0, "entropy": 0.056945288088172674, "epoch": 0.7656427758816837, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.01445537805557251, "kl": 0.007729712524451315, "learning_rate": 2.114334470989761e-06, "loss": 0.0804, "num_tokens": 142100904.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.4998433589935303, "sampling/importance_sampling_ratio/mean": 0.4523466229438782, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8366026878356934, "sampling/sampling_logp_difference/mean": 0.008459349162876606, "step": 1346, "step_time": 74.77798968041316 }, { "clip_ratio/high_max": 0.009647523751482368, "clip_ratio/high_mean": 0.002201954644988291, "clip_ratio/low_mean": 0.001977539053768851, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004179493698757142, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 914.2000122070312, "completions/mean_terminated_length": 536.0, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.06304282182827592, "epoch": 0.7662116040955631, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014687742106616497, "kl": 0.009685556462500244, "learning_rate": 2.1092150170648465e-06, "loss": 0.026, "num_tokens": 142210248.0, "reward": 0.42500001192092896, "reward_std": 0.8129560351371765, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.999891996383667, "sampling/importance_sampling_ratio/mean": 0.42512965202331543, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5717756748199463, "sampling/sampling_logp_difference/mean": 0.008880640380084515, "step": 1347, "step_time": 75.13685992686078 }, { "clip_ratio/high_max": 0.012617279076948762, "clip_ratio/high_mean": 0.004151646833634004, "clip_ratio/low_mean": 0.0007410058024106547, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004892652679700404, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 839.0, "completions/mean_length": 844.6749877929688, "completions/mean_terminated_length": 602.058837890625, "completions/min_length": 419.0, "completions/min_terminated_length": 419.0, "entropy": 0.07744667679071426, "epoch": 0.7667804323094426, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01925349235534668, "kl": 0.012703837710432708, "learning_rate": 2.104095563139932e-06, "loss": -0.0778, "num_tokens": 142318451.0, "reward": 0.824999988079071, "reward_std": 0.9841695427894592, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.601784586906433, "sampling/importance_sampling_ratio/mean": 0.3639231324195862, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9473066329956055, "sampling/sampling_logp_difference/mean": 0.011161512695252895, "step": 1348, "step_time": 74.14426668686792 }, { "clip_ratio/high_max": 0.00818374747177586, "clip_ratio/high_mean": 0.0020100963301956654, "clip_ratio/low_mean": 0.0015625000014551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035725963534787297, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 883.9750366210938, "completions/mean_terminated_length": 743.9500122070312, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.12684078607708216, "epoch": 0.767349260523322, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.043513212352991104, "kl": 0.013210767298005521, "learning_rate": 2.0989761092150174e-06, "loss": -0.1623, "num_tokens": 142408858.0, "reward": 0.7750000357627869, "reward_std": 0.8619447350502014, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.37259840965271, "sampling/importance_sampling_ratio/mean": 0.39729347825050354, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.555673122406006, "sampling/sampling_logp_difference/mean": 0.014674127101898193, "step": 1349, "step_time": 70.65666079125367 }, { "clip_ratio/high_max": 0.015360870689619333, "clip_ratio/high_mean": 0.004915066616376862, "clip_ratio/low_mean": 0.0014338188229885418, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00634888542117551, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 765.6500244140625, "completions/mean_terminated_length": 593.4166870117188, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.08736866107210517, "epoch": 0.7679180887372014, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013901407830417156, "kl": 0.011987186153419316, "learning_rate": 2.0938566552901024e-06, "loss": 0.0134, "num_tokens": 142504372.0, "reward": 1.024999976158142, "reward_std": 0.9195176959037781, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.830623149871826, "sampling/importance_sampling_ratio/mean": 0.3670024275779724, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.465336322784424, "sampling/sampling_logp_difference/mean": 0.011090436950325966, "step": 1350, "step_time": 72.3610460923519 }, { "clip_ratio/high_max": 0.010190355154918507, "clip_ratio/high_mean": 0.0020380710338940844, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0020380710338940844, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 604.0, "completions/mean_length": 827.8250122070312, "completions/mean_terminated_length": 500.86669921875, "completions/min_length": 423.0, "completions/min_terminated_length": 423.0, "entropy": 0.08763806894421577, "epoch": 0.7684869169510807, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.003796446369960904, "kl": 0.01065894978819415, "learning_rate": 2.088737201365188e-06, "loss": 0.0071, "num_tokens": 142604405.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.7975244522094727, "sampling/importance_sampling_ratio/mean": 0.4336318075656891, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8314037322998047, "sampling/sampling_logp_difference/mean": 0.011385971680283546, "step": 1351, "step_time": 72.9964759061113 }, { "clip_ratio/high_max": 0.008764729485847056, "clip_ratio/high_mean": 0.002047063549980521, "clip_ratio/low_mean": 0.0018798827950377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0039269462577067316, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 862.9750366210938, "completions/mean_terminated_length": 563.9285888671875, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.10487956833094358, "epoch": 0.7690557451649602, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.016876675188541412, "kl": 0.01155904692132026, "learning_rate": 2.0836177474402733e-06, "loss": 0.1151, "num_tokens": 142713876.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.1224441528320312, "sampling/importance_sampling_ratio/mean": 0.43348273634910583, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7857112884521484, "sampling/sampling_logp_difference/mean": 0.013219502754509449, "step": 1352, "step_time": 72.39265074767172 }, { "clip_ratio/high_max": 0.016173472860828042, "clip_ratio/high_mean": 0.0061892757075838745, "clip_ratio/low_mean": 0.0012896960179205053, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007478971674572676, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 958.0, "completions/mean_length": 759.7999877929688, "completions/mean_terminated_length": 583.6666870117188, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.08820778969675303, "epoch": 0.7696245733788396, "frac_reward_zero_std": 0.0, "grad_norm": 0.03146493434906006, "kl": 0.009963934309780598, "learning_rate": 2.0784982935153587e-06, "loss": 0.0101, "num_tokens": 142809692.0, "reward": 1.125, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.533726215362549, "sampling/importance_sampling_ratio/mean": 0.5910371541976929, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.882758378982544, "sampling/sampling_logp_difference/mean": 0.011364558711647987, "step": 1353, "step_time": 72.46408994984813 }, { "clip_ratio/high_max": 0.011815745965577662, "clip_ratio/high_mean": 0.0031487455125898123, "clip_ratio/low_mean": 0.0014648437645519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004613589349901304, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 825.0499877929688, "completions/mean_terminated_length": 645.047607421875, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "entropy": 0.07212270749732852, "epoch": 0.770193401592719, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.028079820796847343, "kl": 0.013196651125326753, "learning_rate": 2.0733788395904437e-06, "loss": -0.071, "num_tokens": 142902686.0, "reward": 1.0, "reward_std": 0.9607689380645752, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.6787068843841553, "sampling/importance_sampling_ratio/mean": 0.49044522643089294, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.636317491531372, "sampling/sampling_logp_difference/mean": 0.010675794444978237, "step": 1354, "step_time": 68.5824293976184 }, { "clip_ratio/high_max": 0.005424206960014999, "clip_ratio/high_mean": 0.0014336786116473377, "clip_ratio/low_mean": 0.0016357421991415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030694208544446155, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 979.0, "completions/mean_length": 844.2999877929688, "completions/mean_terminated_length": 724.5, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "entropy": 0.09760987758636475, "epoch": 0.7707622298065984, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01285798940807581, "kl": 0.010765178594738245, "learning_rate": 2.068259385665529e-06, "loss": 0.0049, "num_tokens": 142992418.0, "reward": 1.100000023841858, "reward_std": 0.9554164409637451, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.4313204288482666, "sampling/importance_sampling_ratio/mean": 0.3291487693786621, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6893177032470703, "sampling/sampling_logp_difference/mean": 0.012706195004284382, "step": 1355, "step_time": 73.25028361426666 }, { "clip_ratio/high_max": 0.010274183936417103, "clip_ratio/high_mean": 0.0024053040106082335, "clip_ratio/low_mean": 0.0025390625232830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0049443665775470436, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 852.0, "completions/mean_length": 974.7250366210938, "completions/mean_terminated_length": 695.5, "completions/min_length": 481.0, "completions/min_terminated_length": 481.0, "entropy": 0.1180098233744502, "epoch": 0.7713310580204779, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.007753518410027027, "kl": 0.010584018658846617, "learning_rate": 2.063139931740614e-06, "loss": 0.0439, "num_tokens": 143108751.0, "reward": 0.22500000894069672, "reward_std": 0.5767948627471924, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.3616202771663666, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 1.7827345132827759, "sampling/importance_sampling_ratio/mean": 0.2814037501811981, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.438085913658142, "sampling/sampling_logp_difference/mean": 0.014112072065472603, "step": 1356, "step_time": 76.74339550943114 }, { "clip_ratio/high_max": 0.013503188034519553, "clip_ratio/high_mean": 0.005074853805126622, "clip_ratio/low_mean": 0.002661132784851361, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.007735986670013517, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 856.375, "completions/mean_terminated_length": 629.5882568359375, "completions/min_length": 386.0, "completions/min_terminated_length": 386.0, "entropy": 0.10538925789296627, "epoch": 0.7718998862343572, "frac_reward_zero_std": 0.0, "grad_norm": 0.02313614822924137, "kl": 0.014784321072511375, "learning_rate": 2.0580204778156996e-06, "loss": -0.0381, "num_tokens": 143218294.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.0693352222442627, "sampling/importance_sampling_ratio/mean": 0.3256409466266632, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6231682300567627, "sampling/sampling_logp_difference/mean": 0.013217076659202576, "step": 1357, "step_time": 75.07560919201933 }, { "clip_ratio/high_max": 0.009983202267903835, "clip_ratio/high_mean": 0.0027142831386299804, "clip_ratio/low_mean": 0.0009765624890860636, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036908455804223195, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 921.4500122070312, "completions/mean_terminated_length": 651.0909423828125, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.08793732663616538, "epoch": 0.7724687144482366, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025530347600579262, "kl": 0.00990088254911825, "learning_rate": 2.052901023890785e-06, "loss": 0.0653, "num_tokens": 143316544.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.0407063961029053, "sampling/importance_sampling_ratio/mean": 0.4720449447631836, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0973474979400635, "sampling/sampling_logp_difference/mean": 0.010581472888588905, "step": 1358, "step_time": 70.35493881814182 }, { "clip_ratio/high_max": 0.011323302518576384, "clip_ratio/high_mean": 0.0026365187513874844, "clip_ratio/low_mean": 0.003906250014551915, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.006542768678627908, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 921.0, "completions/mean_length": 950.375, "completions/mean_terminated_length": 696.7777709960938, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "entropy": 0.11736107990145683, "epoch": 0.773037542662116, "frac_reward_zero_std": 0.0, "grad_norm": 0.030411774292588234, "kl": 0.011681885865982622, "learning_rate": 2.0477815699658705e-06, "loss": 0.0393, "num_tokens": 143425847.0, "reward": 0.375, "reward_std": 0.7403221726417542, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290207743644714, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.910526752471924, "sampling/importance_sampling_ratio/mean": 0.4477061629295349, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.5278372764587402, "sampling/sampling_logp_difference/mean": 0.013697382993996143, "step": 1359, "step_time": 75.1315900166519 }, { "clip_ratio/high_max": 0.015021901635918766, "clip_ratio/high_mean": 0.004552941318252124, "clip_ratio/low_mean": 0.001684570306679234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006237511523067951, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 857.0, "completions/mean_length": 874.9750366210938, "completions/mean_terminated_length": 598.2142944335938, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.10797193925827742, "epoch": 0.7736063708759955, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02858634665608406, "kl": 0.011798013059888035, "learning_rate": 2.0426621160409555e-06, "loss": -0.012, "num_tokens": 143533742.0, "reward": 0.6500000357627869, "reward_std": 0.9212589263916016, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.83121919631958, "sampling/importance_sampling_ratio/mean": 0.39689555764198303, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.638751268386841, "sampling/sampling_logp_difference/mean": 0.012260489165782928, "step": 1360, "step_time": 74.54995954036713 }, { "clip_ratio/high_max": 0.0015243901871144772, "clip_ratio/high_mean": 0.0003048780490644276, "clip_ratio/low_mean": 0.0008056640763243195, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001110542125388747, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 796.0, "completions/mean_length": 786.1500244140625, "completions/mean_terminated_length": 464.3529357910156, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.12194172199815512, "epoch": 0.7741751990898749, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.005256331525743008, "kl": 0.012250617844983935, "learning_rate": 2.037542662116041e-06, "loss": 0.0323, "num_tokens": 143638540.0, "reward": 0.824999988079071, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.5151708126068115, "sampling/importance_sampling_ratio/mean": 0.48508143424987793, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.938668251037598, "sampling/sampling_logp_difference/mean": 0.015166327357292175, "step": 1361, "step_time": 71.52706313948147 }, { "clip_ratio/high_max": 0.014524720958434045, "clip_ratio/high_mean": 0.004134953793254681, "clip_ratio/low_mean": 0.0030915335955796763, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007226487505249679, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.0, "completions/mean_length": 834.7000122070312, "completions/mean_terminated_length": 519.2000122070312, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.10840946901589632, "epoch": 0.7747440273037542, "frac_reward_zero_std": 0.0, "grad_norm": 0.016707591712474823, "kl": 0.016625812742859125, "learning_rate": 2.0324232081911263e-06, "loss": 0.0564, "num_tokens": 143745944.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 1.619162678718567, "sampling/importance_sampling_ratio/mean": 0.2783682942390442, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.21648907661438, "sampling/sampling_logp_difference/mean": 0.013855394907295704, "step": 1362, "step_time": 72.39586283662356 }, { "clip_ratio/high_max": 0.013392616296187043, "clip_ratio/high_mean": 0.004418967175297439, "clip_ratio/low_mean": 0.0024902343284338713, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006909201561938971, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 832.0, "completions/mean_length": 796.2000122070312, "completions/mean_terminated_length": 517.7777709960938, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.09716597106307745, "epoch": 0.7753128555176336, "frac_reward_zero_std": 0.0, "grad_norm": 0.024526400491595268, "kl": 0.011312970367725939, "learning_rate": 2.0273037542662118e-06, "loss": -0.0005, "num_tokens": 143844944.0, "reward": 0.75, "reward_std": 0.8987170457839966, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.6956839561462402, "sampling/importance_sampling_ratio/mean": 0.3869382441043854, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8133577108383179, "sampling/sampling_logp_difference/mean": 0.011926032602787018, "step": 1363, "step_time": 70.63943520910107 }, { "clip_ratio/high_max": 0.010778546216897666, "clip_ratio/high_mean": 0.0038861572684254497, "clip_ratio/low_mean": 0.0015244201858877204, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005410577490692958, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 877.9249877929688, "completions/mean_terminated_length": 716.4736938476562, "completions/min_length": 528.0, "completions/min_terminated_length": 528.0, "entropy": 0.10894654970616102, "epoch": 0.7758816837315131, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01500084437429905, "kl": 0.01156267139595002, "learning_rate": 2.0221843003412968e-06, "loss": 0.0097, "num_tokens": 143952973.0, "reward": 0.875, "reward_std": 0.9657599329948425, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.955450415611267, "sampling/importance_sampling_ratio/mean": 0.31152528524398804, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9748852252960205, "sampling/sampling_logp_difference/mean": 0.0140536455437541, "step": 1364, "step_time": 74.78636504756287 }, { "clip_ratio/high_max": 0.01208811838296242, "clip_ratio/high_mean": 0.004151185210503172, "clip_ratio/low_mean": 0.002709960943320766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006861146277515218, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 833.9000244140625, "completions/mean_terminated_length": 661.90478515625, "completions/min_length": 464.0, "completions/min_terminated_length": 464.0, "entropy": 0.1410231739282608, "epoch": 0.7764505119453925, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01953713782131672, "kl": 0.012288974248804152, "learning_rate": 2.0170648464163822e-06, "loss": 0.0288, "num_tokens": 144061401.0, "reward": 0.949999988079071, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.7632858753204346, "sampling/importance_sampling_ratio/mean": 0.3155999183654785, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0731444358825684, "sampling/sampling_logp_difference/mean": 0.016057396307587624, "step": 1365, "step_time": 72.91563883074559 }, { "clip_ratio/high_max": 0.011023811181075871, "clip_ratio/high_mean": 0.005054305453086272, "clip_ratio/low_mean": 0.0012928141441079788, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006347119517158717, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 779.75, "completions/mean_terminated_length": 648.2307739257812, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "entropy": 0.07110442081466317, "epoch": 0.7770193401592719, "frac_reward_zero_std": 0.0, "grad_norm": 0.019917810335755348, "kl": 0.009734276565723121, "learning_rate": 2.0119453924914677e-06, "loss": -0.0337, "num_tokens": 144166623.0, "reward": 1.25, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.9685919284820557, "sampling/importance_sampling_ratio/mean": 0.3389319181442261, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.151163101196289, "sampling/sampling_logp_difference/mean": 0.009930302388966084, "step": 1366, "step_time": 76.896047902992 }, { "clip_ratio/high_max": 0.016081667505204678, "clip_ratio/high_mean": 0.006027903349604458, "clip_ratio/low_mean": 0.001931628939928487, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007959532260429114, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 844.1749877929688, "completions/mean_terminated_length": 664.3500366210938, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.09523110184818506, "epoch": 0.7775881683731513, "frac_reward_zero_std": 0.0, "grad_norm": 0.024605276063084602, "kl": 0.011807392700575292, "learning_rate": 2.006825938566553e-06, "loss": 0.0335, "num_tokens": 144271582.0, "reward": 0.875, "reward_std": 0.9388344883918762, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.463148355484009, "sampling/importance_sampling_ratio/mean": 0.31345438957214355, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.192007064819336, "sampling/sampling_logp_difference/mean": 0.012058037333190441, "step": 1367, "step_time": 76.84211693005636 }, { "clip_ratio/high_max": 0.013667182822246104, "clip_ratio/high_mean": 0.004367845947854221, "clip_ratio/low_mean": 0.0022462492597696837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006614095123950392, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 781.25, "completions/mean_terminated_length": 601.8261108398438, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "entropy": 0.10679132025688887, "epoch": 0.7781569965870307, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027794811874628067, "kl": 0.010643314570188522, "learning_rate": 2.001706484641638e-06, "loss": 0.0705, "num_tokens": 144380880.0, "reward": 0.9000000357627869, "reward_std": 0.8711913824081421, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.1982954740524292, "sampling/importance_sampling_ratio/mean": 0.3436734676361084, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.636373281478882, "sampling/sampling_logp_difference/mean": 0.014260686933994293, "step": 1368, "step_time": 73.91677941102535 }, { "clip_ratio/high_max": 0.012504806159995496, "clip_ratio/high_mean": 0.004213170293951407, "clip_ratio/low_mean": 0.0017578125189174898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005970982776489109, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 885.0, "completions/mean_terminated_length": 596.3077392578125, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.08437930373474956, "epoch": 0.7787258248009101, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015019150450825691, "kl": 0.011114371998701245, "learning_rate": 1.9965870307167235e-06, "loss": 0.0767, "num_tokens": 144485096.0, "reward": 0.550000011920929, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.663604497909546, "sampling/importance_sampling_ratio/mean": 0.4660188555717468, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.406797170639038, "sampling/sampling_logp_difference/mean": 0.011295373551547527, "step": 1369, "step_time": 77.19884596648626 }, { "clip_ratio/high_max": 0.008613448881078511, "clip_ratio/high_mean": 0.002551252953708172, "clip_ratio/low_mean": 0.0033173849660670385, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0058686379343271255, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 855.75, "completions/mean_terminated_length": 628.11767578125, "completions/min_length": 407.0, "completions/min_terminated_length": 407.0, "entropy": 0.11532580293715, "epoch": 0.7792946530147895, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.036204464733600616, "kl": 0.015589422429911792, "learning_rate": 1.991467576791809e-06, "loss": -0.0883, "num_tokens": 144595870.0, "reward": 0.6000000238418579, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.269517183303833, "sampling/importance_sampling_ratio/mean": 0.5970789194107056, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.002948045730591, "sampling/sampling_logp_difference/mean": 0.013759619556367397, "step": 1370, "step_time": 74.68454299098812 }, { "clip_ratio/high_max": 0.012205654784338549, "clip_ratio/high_mean": 0.004498668036831077, "clip_ratio/low_mean": 0.002172851574869128, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006671519426163286, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 888.0750122070312, "completions/mean_terminated_length": 704.1764526367188, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.09247502451762557, "epoch": 0.7798634812286689, "frac_reward_zero_std": 0.0, "grad_norm": 0.0368025116622448, "kl": 0.013396338792517781, "learning_rate": 1.9863481228668944e-06, "loss": -0.076, "num_tokens": 144697713.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.760709762573242, "sampling/importance_sampling_ratio/mean": 0.4178754687309265, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.121958255767822, "sampling/sampling_logp_difference/mean": 0.012387369759380817, "step": 1371, "step_time": 73.32513889158145 }, { "clip_ratio/high_max": 0.014695014106109738, "clip_ratio/high_mean": 0.0057697289157658815, "clip_ratio/low_mean": 0.001351500177406706, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007121229165932164, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 808.0, "completions/mean_length": 801.4249877929688, "completions/mean_terminated_length": 529.388916015625, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.09291462507098913, "epoch": 0.7804323094425484, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02749483659863472, "kl": 0.015586901339702308, "learning_rate": 1.9812286689419794e-06, "loss": 0.2243, "num_tokens": 144808154.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.0179147720336914, "sampling/importance_sampling_ratio/mean": 0.3627505898475647, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.268533229827881, "sampling/sampling_logp_difference/mean": 0.01234792172908783, "step": 1372, "step_time": 76.48319393605925 }, { "clip_ratio/high_max": 0.013216635095886886, "clip_ratio/high_mean": 0.0045666799414902925, "clip_ratio/low_mean": 0.0013427734411379788, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005909453408094123, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 827.0, "completions/mean_length": 779.625, "completions/mean_terminated_length": 558.5238037109375, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.08558572921901941, "epoch": 0.7810011376564278, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01622815430164337, "kl": 0.012524468533229083, "learning_rate": 1.976109215017065e-06, "loss": 0.0286, "num_tokens": 144915659.0, "reward": 0.949999988079071, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.583895683288574, "sampling/importance_sampling_ratio/mean": 0.39218586683273315, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.432424068450928, "sampling/sampling_logp_difference/mean": 0.012146263383328915, "step": 1373, "step_time": 75.72552340384573 }, { "clip_ratio/high_max": 0.01536344294436276, "clip_ratio/high_mean": 0.005446153198136017, "clip_ratio/low_mean": 0.0019175326815457083, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007363685930613428, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 903.0, "completions/mean_length": 695.5, "completions/mean_terminated_length": 537.3333129882812, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.1108288299292326, "epoch": 0.7815699658703071, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019148187711834908, "kl": 0.015507880132645369, "learning_rate": 1.9709897610921503e-06, "loss": -0.0409, "num_tokens": 145020015.0, "reward": 1.274999976158142, "reward_std": 0.9604352116584778, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.33577036857605, "sampling/importance_sampling_ratio/mean": 0.4344298243522644, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.720767021179199, "sampling/sampling_logp_difference/mean": 0.015356919728219509, "step": 1374, "step_time": 72.35632792464457 }, { "clip_ratio/high_max": 0.015202886890619993, "clip_ratio/high_mean": 0.004790677659912035, "clip_ratio/low_mean": 0.0021972656104480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006987943139392883, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 862.0, "completions/mean_length": 887.625, "completions/mean_terminated_length": 634.357177734375, "completions/min_length": 397.0, "completions/min_terminated_length": 397.0, "entropy": 0.10906852316111326, "epoch": 0.7821387940841866, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02127900905907154, "kl": 0.011893283342942595, "learning_rate": 1.9658703071672357e-06, "loss": 0.0111, "num_tokens": 145128832.0, "reward": 0.625, "reward_std": 0.896932065486908, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 1.5640469789505005, "sampling/importance_sampling_ratio/mean": 0.3017110526561737, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.872646689414978, "sampling/sampling_logp_difference/mean": 0.01379802729934454, "step": 1375, "step_time": 74.61642498103902 }, { "clip_ratio/high_max": 0.007836441567633301, "clip_ratio/high_mean": 0.00264202561811544, "clip_ratio/low_mean": 0.0026123047136934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005254330346360803, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 934.0, "completions/mean_length": 819.4000244140625, "completions/mean_terminated_length": 652.0, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "entropy": 0.12300315033644438, "epoch": 0.782707622298066, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014967921189963818, "kl": 0.011733973224181682, "learning_rate": 1.9607508532423207e-06, "loss": -0.0746, "num_tokens": 145236616.0, "reward": 1.100000023841858, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.5348081588745117, "sampling/importance_sampling_ratio/mean": 0.32756131887435913, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.986546039581299, "sampling/sampling_logp_difference/mean": 0.013676205649971962, "step": 1376, "step_time": 75.50623587099835 }, { "clip_ratio/high_max": 0.006659493374172598, "clip_ratio/high_mean": 0.0017663806211203337, "clip_ratio/low_mean": 0.002342422798392363, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004108803404960781, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 935.0, "completions/mean_length": 801.6000366210938, "completions/mean_terminated_length": 637.2174072265625, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.14204016234725714, "epoch": 0.7832764505119454, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.009019244462251663, "kl": 0.013302832609042525, "learning_rate": 1.955631399317406e-06, "loss": 0.0522, "num_tokens": 145342096.0, "reward": 1.125, "reward_std": 0.9919548630714417, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.0557374954223633, "sampling/importance_sampling_ratio/mean": 0.42591628432273865, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8526496887207031, "sampling/sampling_logp_difference/mean": 0.01539462711662054, "step": 1377, "step_time": 73.1958555309102 }, { "clip_ratio/high_max": 0.01726926735136658, "clip_ratio/high_mean": 0.006678036443190649, "clip_ratio/low_mean": 0.0028076171729480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00948565365979448, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 863.0499877929688, "completions/mean_terminated_length": 666.3333129882812, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.1478866646066308, "epoch": 0.7838452787258248, "frac_reward_zero_std": 0.0, "grad_norm": 0.03942440450191498, "kl": 0.011936215218156576, "learning_rate": 1.9505119453924916e-06, "loss": -0.0442, "num_tokens": 145453458.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.398008346557617, "sampling/importance_sampling_ratio/mean": 0.48824048042297363, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.425015926361084, "sampling/sampling_logp_difference/mean": 0.015723904594779015, "step": 1378, "step_time": 75.96957870107144 }, { "clip_ratio/high_max": 0.009738993598148227, "clip_ratio/high_mean": 0.0037275999056873843, "clip_ratio/low_mean": 0.0013916015668655746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00511920137796551, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 888.625, "completions/mean_terminated_length": 685.5625, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "entropy": 0.06815403746441007, "epoch": 0.7844141069397043, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03934136778116226, "kl": 0.008683601045049727, "learning_rate": 1.945392491467577e-06, "loss": -0.0613, "num_tokens": 145562683.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.8355143070220947, "sampling/importance_sampling_ratio/mean": 0.5822598338127136, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.786029577255249, "sampling/sampling_logp_difference/mean": 0.010036606341600418, "step": 1379, "step_time": 76.39149297494441 }, { "clip_ratio/high_max": 0.010215234942734241, "clip_ratio/high_mean": 0.003388825847650878, "clip_ratio/low_mean": 0.0019042968633584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005293122900184244, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 940.0, "completions/mean_length": 880.25, "completions/mean_terminated_length": 581.6923217773438, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.09358878526836634, "epoch": 0.7849829351535836, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021948207169771194, "kl": 0.009980347764212638, "learning_rate": 1.940273037542662e-06, "loss": 0.0207, "num_tokens": 145664309.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.4690001010894775, "sampling/importance_sampling_ratio/mean": 0.3739749789237976, "sampling/importance_sampling_ratio/min": 0.008369180373847485, "sampling/sampling_logp_difference/max": 1.9746730327606201, "sampling/sampling_logp_difference/mean": 0.01142143178731203, "step": 1380, "step_time": 73.25334378704429 }, { "clip_ratio/high_max": 0.011805779649876058, "clip_ratio/high_mean": 0.0029649711796082556, "clip_ratio/low_mean": 0.0016601562456344254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004625127272447571, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 870.2750244140625, "completions/mean_terminated_length": 639.6875, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.10301767010241747, "epoch": 0.785551763367463, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018885469064116478, "kl": 0.010403133346699178, "learning_rate": 1.9351535836177475e-06, "loss": 0.0462, "num_tokens": 145771024.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.590815544128418, "sampling/importance_sampling_ratio/mean": 0.5208168625831604, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9859378337860107, "sampling/sampling_logp_difference/mean": 0.011989383958280087, "step": 1381, "step_time": 74.50178634840995 }, { "clip_ratio/high_max": 0.009474529535509646, "clip_ratio/high_mean": 0.003160505002597347, "clip_ratio/low_mean": 0.0032226562034338713, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.0063831612933427095, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 924.9249877929688, "completions/mean_terminated_length": 719.1538696289062, "completions/min_length": 377.0, "completions/min_terminated_length": 377.0, "entropy": 0.12275072280317545, "epoch": 0.7861205915813424, "frac_reward_zero_std": 0.0, "grad_norm": 0.0635451003909111, "kl": 0.011739021225366741, "learning_rate": 1.930034129692833e-06, "loss": -0.1467, "num_tokens": 145878821.0, "reward": 0.574999988079071, "reward_std": 0.873762845993042, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.8487558364868164, "sampling/importance_sampling_ratio/mean": 0.5847955942153931, "sampling/importance_sampling_ratio/min": 0.009496710263192654, "sampling/sampling_logp_difference/max": 1.9665091037750244, "sampling/sampling_logp_difference/mean": 0.013408188708126545, "step": 1382, "step_time": 75.07931914832443 }, { "clip_ratio/high_max": 0.01005073927808553, "clip_ratio/high_mean": 0.0024881003046175465, "clip_ratio/low_mean": 0.0006347656453726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003122865928162355, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 779.0, "completions/mean_length": 789.25, "completions/mean_terminated_length": 502.3333435058594, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.10094770882278681, "epoch": 0.7866894197952219, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009169720113277435, "kl": 0.00990298215765506, "learning_rate": 1.9249146757679183e-06, "loss": -0.0124, "num_tokens": 145986223.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 0.8450567722320557, "sampling/importance_sampling_ratio/mean": 0.25336748361587524, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.56270170211792, "sampling/sampling_logp_difference/mean": 0.013172957114875317, "step": 1383, "step_time": 74.10220219171606 }, { "clip_ratio/high_max": 0.008583113667555153, "clip_ratio/high_mean": 0.0027841444170917384, "clip_ratio/low_mean": 0.0013183593764551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00410250379354693, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 886.5750122070312, "completions/mean_terminated_length": 601.1538696289062, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.09525041747838259, "epoch": 0.7872582480091013, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01680304855108261, "kl": 0.011339098156895489, "learning_rate": 1.9197952218430033e-06, "loss": 0.0326, "num_tokens": 146093534.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.2145516872406006, "sampling/importance_sampling_ratio/mean": 0.37909403443336487, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.009136438369751, "sampling/sampling_logp_difference/mean": 0.012019495479762554, "step": 1384, "step_time": 72.72794202319346 }, { "clip_ratio/high_max": 0.007826252491213381, "clip_ratio/high_mean": 0.002035621553659439, "clip_ratio/low_mean": 0.0015380859440483619, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003573707494069822, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 820.0, "completions/mean_length": 832.9500122070312, "completions/mean_terminated_length": 514.5333862304688, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.09058926720172167, "epoch": 0.7878270762229806, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015908539295196533, "kl": 0.01291456469334662, "learning_rate": 1.9146757679180888e-06, "loss": 0.0095, "num_tokens": 146198708.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.4866833686828613, "sampling/importance_sampling_ratio/mean": 0.31460410356521606, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.023587226867676, "sampling/sampling_logp_difference/mean": 0.012502907775342464, "step": 1385, "step_time": 75.22813886590302 }, { "clip_ratio/high_max": 0.013136960566043854, "clip_ratio/high_mean": 0.004648450136301108, "clip_ratio/low_mean": 0.002527705655666068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007176155777415261, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 844.0250244140625, "completions/mean_terminated_length": 664.0499877929688, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.11704358831048012, "epoch": 0.78839590443686, "frac_reward_zero_std": 0.0, "grad_norm": 0.016147486865520477, "kl": 0.012217935523949564, "learning_rate": 1.9095563139931742e-06, "loss": 0.0089, "num_tokens": 146295181.0, "reward": 0.925000011920929, "reward_std": 0.9710555076599121, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.910324811935425, "sampling/importance_sampling_ratio/mean": 0.42217960953712463, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.911831021308899, "sampling/sampling_logp_difference/mean": 0.01348027028143406, "step": 1386, "step_time": 72.06548670073971 }, { "clip_ratio/high_max": 0.014597439672797918, "clip_ratio/high_mean": 0.004045659996336326, "clip_ratio/low_mean": 0.001806640619179234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00585230061551556, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 931.0, "completions/mean_length": 919.3500366210938, "completions/mean_terminated_length": 605.4000244140625, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "entropy": 0.1732981950044632, "epoch": 0.7889647326507395, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.017342571169137955, "kl": 0.010529191116802394, "learning_rate": 1.9044368600682594e-06, "loss": 0.0137, "num_tokens": 146407571.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.573828935623169, "sampling/importance_sampling_ratio/mean": 0.3210883140563965, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6463267803192139, "sampling/sampling_logp_difference/mean": 0.0166267491877079, "step": 1387, "step_time": 79.7274583929684 }, { "clip_ratio/high_max": 0.012645127484574914, "clip_ratio/high_mean": 0.004358205056632869, "clip_ratio/low_mean": 0.0005859375014551915, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004944142478052527, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 793.75, "completions/mean_terminated_length": 512.3333129882812, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.07688467530533671, "epoch": 0.7895335608646189, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.012437013909220695, "kl": 0.01292136829579249, "learning_rate": 1.8993174061433449e-06, "loss": -0.0575, "num_tokens": 146509513.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.8912808895111084, "sampling/importance_sampling_ratio/mean": 0.30898517370224, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3434832096099854, "sampling/sampling_logp_difference/mean": 0.011224405840039253, "step": 1388, "step_time": 73.63260276988149 }, { "clip_ratio/high_max": 0.013324639352504164, "clip_ratio/high_mean": 0.003916959947673604, "clip_ratio/low_mean": 0.0009277343888243195, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004844694369239733, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 977.0, "completions/mean_length": 825.4500122070312, "completions/mean_terminated_length": 663.0, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.09124117903411388, "epoch": 0.7901023890784983, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018247758969664574, "kl": 0.012704712338745594, "learning_rate": 1.8941979522184303e-06, "loss": -0.0514, "num_tokens": 146612395.0, "reward": 0.925000011920929, "reward_std": 0.9167249798774719, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.935625433921814, "sampling/importance_sampling_ratio/mean": 0.33521568775177, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1943655014038086, "sampling/sampling_logp_difference/mean": 0.01286386139690876, "step": 1389, "step_time": 73.55485829384997 }, { "clip_ratio/high_max": 0.014767017564736307, "clip_ratio/high_mean": 0.004224936361424625, "clip_ratio/low_mean": 0.0016357422136934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005860678502358496, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 886.4500122070312, "completions/mean_terminated_length": 600.7692260742188, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.10430222377181053, "epoch": 0.7906712172923777, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013372387737035751, "kl": 0.013328933040611446, "learning_rate": 1.8890784982935153e-06, "loss": 0.0669, "num_tokens": 146719997.0, "reward": 0.574999988079071, "reward_std": 0.8737629055976868, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.8591618537902832, "sampling/importance_sampling_ratio/mean": 0.20603342354297638, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1033823490142822, "sampling/sampling_logp_difference/mean": 0.012348301708698273, "step": 1390, "step_time": 74.84527729591355 }, { "clip_ratio/high_max": 0.008997280732728541, "clip_ratio/high_mean": 0.00219298513547983, "clip_ratio/low_mean": 0.00034179688191215973, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0025347820192109793, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 805.9249877929688, "completions/mean_terminated_length": 564.8947143554688, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.09562394674867392, "epoch": 0.7912400455062572, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011977402493357658, "kl": 0.01286507531767711, "learning_rate": 1.8839590443686005e-06, "loss": -0.0411, "num_tokens": 146829650.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.585900902748108, "sampling/importance_sampling_ratio/mean": 0.2893064618110657, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.619843006134033, "sampling/sampling_logp_difference/mean": 0.013243592344224453, "step": 1391, "step_time": 76.30735944374464 }, { "clip_ratio/high_max": 0.013446697732433677, "clip_ratio/high_mean": 0.004265082359779626, "clip_ratio/low_mean": 0.0029785156511934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007243597821798176, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 940.0, "completions/mean_length": 855.4000244140625, "completions/mean_terminated_length": 602.5, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.10810105409473181, "epoch": 0.7918088737201365, "frac_reward_zero_std": 0.0, "grad_norm": 0.04388894513249397, "kl": 0.012697174679487944, "learning_rate": 1.878839590443686e-06, "loss": -0.0571, "num_tokens": 146932586.0, "reward": 0.75, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.5507900714874268, "sampling/importance_sampling_ratio/mean": 0.5175482034683228, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.661713123321533, "sampling/sampling_logp_difference/mean": 0.013371369801461697, "step": 1392, "step_time": 72.06914869020693 }, { "clip_ratio/high_max": 0.019250266021117568, "clip_ratio/high_mean": 0.005575158356805332, "clip_ratio/low_mean": 0.0028036857838742435, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008378844300750643, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 819.25, "completions/mean_terminated_length": 614.5, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.09493549074977636, "epoch": 0.7923777019340159, "frac_reward_zero_std": 0.0, "grad_norm": 0.03203502669930458, "kl": 0.010367736453190446, "learning_rate": 1.8737201365187712e-06, "loss": 0.0901, "num_tokens": 147034012.0, "reward": 0.7750000357627869, "reward_std": 0.8911962509155273, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.6420719623565674, "sampling/importance_sampling_ratio/mean": 0.44477301836013794, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.673487424850464, "sampling/sampling_logp_difference/mean": 0.01284459512680769, "step": 1393, "step_time": 72.46670965501107 }, { "clip_ratio/high_max": 0.015263924840837717, "clip_ratio/high_mean": 0.005688110497430898, "clip_ratio/low_mean": 0.0016845702994032763, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007372680818662047, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 847.6500244140625, "completions/mean_terminated_length": 632.1111450195312, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.12637685984373093, "epoch": 0.7929465301478953, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030792493373155594, "kl": 0.01359536498785019, "learning_rate": 1.8686006825938566e-06, "loss": -0.0196, "num_tokens": 147145062.0, "reward": 0.8500000238418579, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.0974907875061035, "sampling/importance_sampling_ratio/mean": 0.33426183462142944, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.355440139770508, "sampling/sampling_logp_difference/mean": 0.015481481328606606, "step": 1394, "step_time": 73.94917182112113 }, { "clip_ratio/high_max": 0.013422711635939777, "clip_ratio/high_mean": 0.005463740788400173, "clip_ratio/low_mean": 0.0019913022879336495, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007455042912624776, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 990.0, "completions/mean_length": 756.5250244140625, "completions/mean_terminated_length": 641.8928833007812, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.07619418390095234, "epoch": 0.7935153583617748, "frac_reward_zero_std": 0.0, "grad_norm": 0.01984211429953575, "kl": 0.012573605519719422, "learning_rate": 1.8634812286689419e-06, "loss": -0.0885, "num_tokens": 147237715.0, "reward": 1.25, "reward_std": 0.8987170457839966, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.6404340267181396, "sampling/importance_sampling_ratio/mean": 0.35812944173812866, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8812649250030518, "sampling/sampling_logp_difference/mean": 0.011560454033315182, "step": 1395, "step_time": 71.1002696682699 }, { "clip_ratio/high_max": 0.015765958931297064, "clip_ratio/high_mean": 0.004055489363963716, "clip_ratio/low_mean": 0.002514648425858468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00657013786258176, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 803.0, "completions/mean_length": 881.5499877929688, "completions/mean_terminated_length": 549.1666870117188, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.08961573150008917, "epoch": 0.7940841865756542, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020041372627019882, "kl": 0.011715068016201258, "learning_rate": 1.8583617747440273e-06, "loss": 0.0989, "num_tokens": 147343537.0, "reward": 0.574999988079071, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.8622536659240723, "sampling/importance_sampling_ratio/mean": 0.44566431641578674, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.235924005508423, "sampling/sampling_logp_difference/mean": 0.012803330086171627, "step": 1396, "step_time": 74.48768087266944 }, { "clip_ratio/high_max": 0.007883279147790745, "clip_ratio/high_mean": 0.0021334635275707114, "clip_ratio/low_mean": 0.0013916015632275958, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035250650689704344, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 852.0, "completions/mean_length": 816.125, "completions/mean_terminated_length": 504.3125, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.12425256054848433, "epoch": 0.7946530147895335, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011932899244129658, "kl": 0.01408401201479137, "learning_rate": 1.8532423208191125e-06, "loss": 0.018, "num_tokens": 147442246.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.6043192148208618, "sampling/importance_sampling_ratio/mean": 0.2912236154079437, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2321009635925293, "sampling/sampling_logp_difference/mean": 0.014788168482482433, "step": 1397, "step_time": 69.59276773897 }, { "clip_ratio/high_max": 0.012391917174682021, "clip_ratio/high_mean": 0.0027234815061092377, "clip_ratio/low_mean": 0.0007616387083544396, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034851201489800587, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 891.6000366210938, "completions/mean_terminated_length": 435.5555725097656, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.15110267139971256, "epoch": 0.7952218430034129, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009752240963280201, "kl": 0.01244006329216063, "learning_rate": 1.848122866894198e-06, "loss": 0.0377, "num_tokens": 147549998.0, "reward": 0.4000000059604645, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.6275107860565186, "sampling/importance_sampling_ratio/mean": 0.4184952676296234, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0638608932495117, "sampling/sampling_logp_difference/mean": 0.015541818924248219, "step": 1398, "step_time": 81.67209033179097 }, { "clip_ratio/high_max": 0.005170239426661283, "clip_ratio/high_mean": 0.0010340478766011074, "clip_ratio/low_mean": 0.0007245208471431397, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017585687237442471, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 877.0, "completions/mean_length": 976.5250244140625, "completions/mean_terminated_length": 752.7142944335938, "completions/min_length": 625.0, "completions/min_terminated_length": 625.0, "entropy": 0.10234206449240446, "epoch": 0.7957906712172924, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.01596798747777939, "kl": 0.008887789386790246, "learning_rate": 1.8430034129692832e-06, "loss": -0.0315, "num_tokens": 147657155.0, "reward": 0.2750000059604645, "reward_std": 0.6400120258331299, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.153343439102173, "sampling/importance_sampling_ratio/mean": 0.37924930453300476, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0018677711486816, "sampling/sampling_logp_difference/mean": 0.01163490954786539, "step": 1399, "step_time": 74.5875076293014 }, { "clip_ratio/high_max": 0.008913784811738878, "clip_ratio/high_mean": 0.002676755902939476, "clip_ratio/low_mean": 0.0010379015675425762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003714657432283275, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 920.0, "completions/mean_length": 937.0750122070312, "completions/mean_terminated_length": 734.25, "completions/min_length": 509.0, "completions/min_terminated_length": 509.0, "entropy": 0.05531563702970743, "epoch": 0.7963594994311718, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01185261644423008, "kl": 0.008200038166251034, "learning_rate": 1.8378839590443686e-06, "loss": 0.042, "num_tokens": 147767790.0, "reward": 0.5250000357627869, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.1959755420684814, "sampling/importance_sampling_ratio/mean": 0.3329133093357086, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5255120992660522, "sampling/sampling_logp_difference/mean": 0.009111645631492138, "step": 1400, "step_time": 75.33163832197897 }, { "clip_ratio/high_max": 0.007494350080378354, "clip_ratio/high_mean": 0.00224783019075403, "clip_ratio/low_mean": 0.0021484375265572453, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00439626764273271, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 834.0, "completions/mean_length": 917.5250244140625, "completions/mean_terminated_length": 598.1000366210938, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.09490978717803955, "epoch": 0.7969283276450512, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03341546654701233, "kl": 0.010423607716802508, "learning_rate": 1.832764505119454e-06, "loss": 0.0062, "num_tokens": 147872699.0, "reward": 0.4749999940395355, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.812208414077759, "sampling/importance_sampling_ratio/mean": 0.5057646036148071, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8408541679382324, "sampling/sampling_logp_difference/mean": 0.011442530900239944, "step": 1401, "step_time": 71.4571102282498 }, { "clip_ratio/high_max": 0.010337274201447144, "clip_ratio/high_mean": 0.0034157058980781585, "clip_ratio/low_mean": 0.001992131379665807, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0054078372195363045, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 961.0, "completions/mean_length": 718.8250122070312, "completions/mean_terminated_length": 535.719970703125, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.10468732006847858, "epoch": 0.7974971558589306, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02484137937426567, "kl": 0.01727665343787521, "learning_rate": 1.8276450511945393e-06, "loss": 0.128, "num_tokens": 147967196.0, "reward": 1.1750000715255737, "reward_std": 0.9577617645263672, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.8462841510772705, "sampling/importance_sampling_ratio/mean": 0.5095456838607788, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7948338985443115, "sampling/sampling_logp_difference/mean": 0.013709008693695068, "step": 1402, "step_time": 72.734350820072 }, { "clip_ratio/high_max": 0.015119841205887496, "clip_ratio/high_mean": 0.004805543925613165, "clip_ratio/low_mean": 0.0017333984615106601, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00653894228162244, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 963.0, "completions/mean_length": 880.6500244140625, "completions/mean_terminated_length": 641.7333374023438, "completions/min_length": 407.0, "completions/min_terminated_length": 407.0, "entropy": 0.10397626366466284, "epoch": 0.79806598407281, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023443592712283134, "kl": 0.014982619439251721, "learning_rate": 1.8225255972696247e-06, "loss": 0.1356, "num_tokens": 148077878.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.9035100936889648, "sampling/importance_sampling_ratio/mean": 0.3371828496456146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8665103912353516, "sampling/sampling_logp_difference/mean": 0.013512680307030678, "step": 1403, "step_time": 75.84620233182795 }, { "clip_ratio/high_max": 0.007321379845961928, "clip_ratio/high_mean": 0.0020509284804575145, "clip_ratio/low_mean": 0.0011718750101863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032228035561274737, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 956.25, "completions/mean_terminated_length": 777.6364135742188, "completions/min_length": 565.0, "completions/min_terminated_length": 565.0, "entropy": 0.06272945692762733, "epoch": 0.7986348122866894, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.029223866760730743, "kl": 0.010032640711870044, "learning_rate": 1.81740614334471e-06, "loss": -0.1748, "num_tokens": 148189280.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.5581741333007812, "sampling/importance_sampling_ratio/mean": 0.5220065712928772, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7894442081451416, "sampling/sampling_logp_difference/mean": 0.009446315467357635, "step": 1404, "step_time": 75.56402534293011 }, { "clip_ratio/high_max": 0.013049503701040521, "clip_ratio/high_mean": 0.004286783259885851, "clip_ratio/low_mean": 0.0016601562674622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005946939520072192, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 821.7000122070312, "completions/mean_terminated_length": 672.1739501953125, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "entropy": 0.10147102642804384, "epoch": 0.7992036405005688, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03378886729478836, "kl": 0.01707070879638195, "learning_rate": 1.8122866894197953e-06, "loss": 0.0842, "num_tokens": 148291108.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.9160131216049194, "sampling/importance_sampling_ratio/mean": 0.39023494720458984, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8591365814208984, "sampling/sampling_logp_difference/mean": 0.01344483345746994, "step": 1405, "step_time": 72.92924218298867 }, { "clip_ratio/high_max": 0.011170863115694374, "clip_ratio/high_mean": 0.0033089438802562654, "clip_ratio/low_mean": 0.0023508922204200644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0056598359369672835, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 874.9249877929688, "completions/mean_terminated_length": 626.4666748046875, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.10174027644097805, "epoch": 0.7997724687144482, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04280208423733711, "kl": 0.015823492081835866, "learning_rate": 1.8071672354948806e-06, "loss": -0.0112, "num_tokens": 148396881.0, "reward": 0.6000000238418579, "reward_std": 0.8412445187568665, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.951819658279419, "sampling/importance_sampling_ratio/mean": 0.47918716073036194, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9743099212646484, "sampling/sampling_logp_difference/mean": 0.013005462475121021, "step": 1406, "step_time": 76.26824181783013 }, { "clip_ratio/high_max": 0.011750499485060573, "clip_ratio/high_mean": 0.0025893173296935856, "clip_ratio/low_mean": 0.0022949218546273187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004884239111561328, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 827.2999877929688, "completions/mean_terminated_length": 532.25, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.08251560991629958, "epoch": 0.8003412969283277, "frac_reward_zero_std": 0.0, "grad_norm": 0.036700934171676636, "kl": 0.014799094176851213, "learning_rate": 1.802047781569966e-06, "loss": 0.0176, "num_tokens": 148503869.0, "reward": 0.6500000357627869, "reward_std": 0.8638019561767578, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.6554412841796875, "sampling/importance_sampling_ratio/mean": 0.4241601526737213, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4620161056518555, "sampling/sampling_logp_difference/mean": 0.010758582502603531, "step": 1407, "step_time": 76.15914268046618 }, { "clip_ratio/high_max": 0.017417105846107006, "clip_ratio/high_mean": 0.007544905289250892, "clip_ratio/low_mean": 0.0027380719839129597, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01028297730954364, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 747.5750122070312, "completions/mean_terminated_length": 581.719970703125, "completions/min_length": 159.0, "completions/min_terminated_length": 159.0, "entropy": 0.15813343785703182, "epoch": 0.800910125142207, "frac_reward_zero_std": 0.0, "grad_norm": 0.02631322853267193, "kl": 0.014409018564037979, "learning_rate": 1.7969283276450512e-06, "loss": -0.0367, "num_tokens": 148603500.0, "reward": 1.149999976158142, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.216191053390503, "sampling/importance_sampling_ratio/mean": 0.3326674997806549, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8376922607421875, "sampling/sampling_logp_difference/mean": 0.01785561442375183, "step": 1408, "step_time": 71.4584159902297 }, { "clip_ratio/high_max": 0.01270487706642598, "clip_ratio/high_mean": 0.0039185194473247975, "clip_ratio/low_mean": 0.001741950836731121, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0056604702840559185, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 979.0, "completions/mean_length": 925.8500366210938, "completions/mean_terminated_length": 743.5714721679688, "completions/min_length": 512.0, "completions/min_terminated_length": 512.0, "entropy": 0.1377724213525653, "epoch": 0.8014789533560864, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025836896151304245, "kl": 0.010405745881143957, "learning_rate": 1.7918088737201367e-06, "loss": -0.0256, "num_tokens": 148711006.0, "reward": 0.5, "reward_std": 0.7510676383972168, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 1.7107776403427124, "sampling/importance_sampling_ratio/mean": 0.5307317972183228, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.045850992202759, "sampling/sampling_logp_difference/mean": 0.014346747659146786, "step": 1409, "step_time": 78.3558902675286 }, { "clip_ratio/high_max": 0.012328541022725403, "clip_ratio/high_mean": 0.0040519111207686365, "clip_ratio/low_mean": 0.0011555248347576708, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005207435955526307, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 886.0, "completions/mean_length": 866.0750122070312, "completions/mean_terminated_length": 652.4117431640625, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "entropy": 0.0901190284639597, "epoch": 0.8020477815699659, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01955922320485115, "kl": 0.010931561992038041, "learning_rate": 1.7866894197952219e-06, "loss": -0.0267, "num_tokens": 148809833.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.8472578525543213, "sampling/importance_sampling_ratio/mean": 0.3490575850009918, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8148083686828613, "sampling/sampling_logp_difference/mean": 0.011560793034732342, "step": 1410, "step_time": 73.97077856748365 }, { "clip_ratio/high_max": 0.009538903250358999, "clip_ratio/high_mean": 0.0031446728680748492, "clip_ratio/low_mean": 0.0019290566197014414, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005073729436844587, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 927.0, "completions/mean_length": 835.2999877929688, "completions/mean_terminated_length": 664.5714111328125, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.08364136656746268, "epoch": 0.8026166097838453, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023473257198929787, "kl": 0.009305057465098798, "learning_rate": 1.7815699658703073e-06, "loss": 0.0715, "num_tokens": 148914389.0, "reward": 0.875, "reward_std": 0.9111137390136719, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.8434808254241943, "sampling/importance_sampling_ratio/mean": 0.5855616927146912, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.355440616607666, "sampling/sampling_logp_difference/mean": 0.011440099216997623, "step": 1411, "step_time": 76.48984919022769 }, { "clip_ratio/high_max": 0.009057465475052595, "clip_ratio/high_mean": 0.0021559937595156953, "clip_ratio/low_mean": 0.0019042968724534148, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004060290608322248, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 842.1000366210938, "completions/mean_terminated_length": 569.25, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.11203516740351915, "epoch": 0.8031854379977247, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022934112697839737, "kl": 0.010964662244077772, "learning_rate": 1.7764505119453925e-06, "loss": 0.1109, "num_tokens": 149024049.0, "reward": 0.75, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.155613660812378, "sampling/importance_sampling_ratio/mean": 0.487935870885849, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7123374938964844, "sampling/sampling_logp_difference/mean": 0.014444045722484589, "step": 1412, "step_time": 73.88287693192251 }, { "clip_ratio/high_max": 0.016813393449410796, "clip_ratio/high_mean": 0.006440605851821601, "clip_ratio/low_mean": 0.0019531249708961695, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00839373079361394, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 795.5250244140625, "completions/mean_terminated_length": 588.8095092773438, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.1311267064884305, "epoch": 0.8037542662116041, "frac_reward_zero_std": 0.0, "grad_norm": 0.03257022053003311, "kl": 0.0151988307479769, "learning_rate": 1.771331058020478e-06, "loss": -0.1809, "num_tokens": 149122814.0, "reward": 1.0500000715255737, "reward_std": 1.0114725828170776, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.7456533908843994, "sampling/importance_sampling_ratio/mean": 0.4907439649105072, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.307950973510742, "sampling/sampling_logp_difference/mean": 0.015018552541732788, "step": 1413, "step_time": 75.10114879510365 }, { "clip_ratio/high_max": 0.01691499981097877, "clip_ratio/high_mean": 0.005326194630470127, "clip_ratio/low_mean": 0.0011635099435807206, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00648970459587872, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 904.0, "completions/mean_length": 669.8250122070312, "completions/mean_terminated_length": 499.2962951660156, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.11140298191457987, "epoch": 0.8043230944254836, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.027406664565205574, "kl": 0.016375187318772078, "learning_rate": 1.7662116040955632e-06, "loss": -0.0758, "num_tokens": 149229879.0, "reward": 1.25, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.5599112510681152, "sampling/importance_sampling_ratio/mean": 0.3676071763038635, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9780845642089844, "sampling/sampling_logp_difference/mean": 0.01573430560529232, "step": 1414, "step_time": 72.86679751891643 }, { "clip_ratio/high_max": 0.01695026137167588, "clip_ratio/high_mean": 0.005063806514954194, "clip_ratio/low_mean": 0.0014713541695527965, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006535160820931196, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 722.2750244140625, "completions/mean_terminated_length": 541.239990234375, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.09874852374196053, "epoch": 0.8048919226393629, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03546459227800369, "kl": 0.015783142298460007, "learning_rate": 1.7610921501706486e-06, "loss": 0.1035, "num_tokens": 149333746.0, "reward": 1.149999976158142, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.465972423553467, "sampling/importance_sampling_ratio/mean": 0.30821752548217773, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.438126564025879, "sampling/sampling_logp_difference/mean": 0.014144393615424633, "step": 1415, "step_time": 75.91583460313268 }, { "clip_ratio/high_max": 0.012649052427150309, "clip_ratio/high_mean": 0.004054964912938885, "clip_ratio/low_mean": 0.0018705005495576188, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005925465520704165, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 855.5250244140625, "completions/mean_terminated_length": 602.8125, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.08588123507797718, "epoch": 0.8054607508532423, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03306153416633606, "kl": 0.011634406575467438, "learning_rate": 1.7559726962457338e-06, "loss": 0.0075, "num_tokens": 149444119.0, "reward": 0.75, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.481576681137085, "sampling/importance_sampling_ratio/mean": 0.5582283139228821, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.462096691131592, "sampling/sampling_logp_difference/mean": 0.01153661124408245, "step": 1416, "step_time": 74.03169350815006 }, { "clip_ratio/high_max": 0.01002571068238467, "clip_ratio/high_mean": 0.0034606807894306257, "clip_ratio/low_mean": 0.002259947768834536, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005720628483686596, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 975.0, "completions/mean_length": 914.7250366210938, "completions/mean_terminated_length": 732.6000366210938, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.11141864862293005, "epoch": 0.8060295790671217, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.046444717794656754, "kl": 0.009468398988246918, "learning_rate": 1.7508532423208193e-06, "loss": -0.1855, "num_tokens": 149552596.0, "reward": 0.625, "reward_std": 0.8678739070892334, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.4565417766571045, "sampling/importance_sampling_ratio/mean": 0.4777822494506836, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2991690635681152, "sampling/sampling_logp_difference/mean": 0.012733821757137775, "step": 1417, "step_time": 74.29140177322552 }, { "clip_ratio/high_max": 0.015221664682030678, "clip_ratio/high_mean": 0.0033640660694800317, "clip_ratio/low_mean": 0.0005923440166952787, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003956410015234724, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 751.125, "completions/mean_terminated_length": 449.52630615234375, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.12254157196730375, "epoch": 0.8065984072810012, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.028765399008989334, "kl": 0.018447503331117332, "learning_rate": 1.7457337883959045e-06, "loss": -0.0099, "num_tokens": 149649649.0, "reward": 0.9000000357627869, "reward_std": 0.9818872809410095, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.6790473461151123, "sampling/importance_sampling_ratio/mean": 0.553094744682312, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9263811111450195, "sampling/sampling_logp_difference/mean": 0.014257139526307583, "step": 1418, "step_time": 72.18061144114472 }, { "clip_ratio/high_max": 0.016218858771026134, "clip_ratio/high_mean": 0.0064712868188507855, "clip_ratio/low_mean": 0.0021880328276893124, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008659319661092013, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 814.0, "completions/mean_length": 681.7750244140625, "completions/mean_terminated_length": 497.5000305175781, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.10392349492758512, "epoch": 0.8071672354948806, "frac_reward_zero_std": 0.0, "grad_norm": 0.06486238539218903, "kl": 0.016527951578609645, "learning_rate": 1.74061433447099e-06, "loss": -0.1094, "num_tokens": 149745696.0, "reward": 1.1750000715255737, "reward_std": 0.9306049942970276, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057363510131836, "sampling/importance_sampling_ratio/max": 2.636040210723877, "sampling/importance_sampling_ratio/mean": 0.39797207713127136, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.15324330329895, "sampling/sampling_logp_difference/mean": 0.014660230837762356, "step": 1419, "step_time": 69.06986371078528 }, { "clip_ratio/high_max": 0.010469829256180674, "clip_ratio/high_mean": 0.002093965875246795, "clip_ratio/low_mean": 0.00019531250291038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002289278378157178, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 578.375, "completions/mean_terminated_length": 449.0, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.12719549145549536, "epoch": 0.8077360637087599, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.00855167955160141, "kl": 0.013721996918320656, "learning_rate": 1.7354948805460752e-06, "loss": -0.0291, "num_tokens": 149847599.0, "reward": 1.5500000715255737, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.7749999761581421, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 1.793249249458313, "sampling/importance_sampling_ratio/mean": 0.3901363015174866, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.112637519836426, "sampling/sampling_logp_difference/mean": 0.017064590007066727, "step": 1420, "step_time": 69.43155111093074 }, { "clip_ratio/high_max": 0.006387162487953901, "clip_ratio/high_mean": 0.0012774325150530785, "clip_ratio/low_mean": 0.0012939453226863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0025713778159115463, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 789.0, "completions/mean_length": 983.625, "completions/mean_terminated_length": 620.25, "completions/min_length": 453.0, "completions/min_terminated_length": 453.0, "entropy": 0.10425902716815472, "epoch": 0.8083048919226393, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.00779173756018281, "kl": 0.010381317581050098, "learning_rate": 1.7303754266211606e-06, "loss": 0.0381, "num_tokens": 149955800.0, "reward": 0.20000000298023224, "reward_std": 0.6076436042785645, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.3038218021392822, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.3294529914855957, "sampling/importance_sampling_ratio/mean": 0.29465943574905396, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7178328037261963, "sampling/sampling_logp_difference/mean": 0.01176925003528595, "step": 1421, "step_time": 75.17384352488443 }, { "clip_ratio/high_max": 0.012471861322410405, "clip_ratio/high_mean": 0.003501616040011868, "clip_ratio/low_mean": 0.0039246622764039785, "clip_ratio/low_min": 0.0003869968932121992, "clip_ratio/region_mean": 0.007426278374623507, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 863.0250244140625, "completions/mean_terminated_length": 621.5625, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.17768876440823078, "epoch": 0.8088737201365188, "frac_reward_zero_std": 0.0, "grad_norm": 0.031660743057727814, "kl": 0.014474470051936805, "learning_rate": 1.7252559726962458e-06, "loss": 0.024, "num_tokens": 150066873.0, "reward": 0.625, "reward_std": 0.8378085494041443, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.440887451171875, "sampling/importance_sampling_ratio/mean": 0.4408000111579895, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.856168508529663, "sampling/sampling_logp_difference/mean": 0.017146596685051918, "step": 1422, "step_time": 74.81761767109856 }, { "clip_ratio/high_max": 0.011662997480016202, "clip_ratio/high_mean": 0.0035984882852062583, "clip_ratio/low_mean": 0.0017333984542347025, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005331886874046177, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 808.25, "completions/mean_terminated_length": 631.727294921875, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.11809198185801506, "epoch": 0.8094425483503982, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022888051345944405, "kl": 0.011850361246615648, "learning_rate": 1.7201365187713312e-06, "loss": 0.0069, "num_tokens": 150163707.0, "reward": 1.100000023841858, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.4724247455596924, "sampling/importance_sampling_ratio/mean": 0.44087982177734375, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3209800720214844, "sampling/sampling_logp_difference/mean": 0.013437521643936634, "step": 1423, "step_time": 72.72436320129782 }, { "clip_ratio/high_max": 0.018062805524095893, "clip_ratio/high_mean": 0.004774378932779655, "clip_ratio/low_mean": 0.0026123046700377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007386683602817357, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 770.7250366210938, "completions/mean_terminated_length": 517.4500122070312, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.12624671775847673, "epoch": 0.8100113765642776, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.020954163745045662, "kl": 0.02124081098008901, "learning_rate": 1.7150170648464165e-06, "loss": 0.0476, "num_tokens": 150258568.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9436941146850586, "sampling/importance_sampling_ratio/mean": 0.5209187269210815, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.270298957824707, "sampling/sampling_logp_difference/mean": 0.01688847877085209, "step": 1424, "step_time": 166.60038918093778 }, { "clip_ratio/high_max": 0.011095629539340734, "clip_ratio/high_mean": 0.0037253568589221686, "clip_ratio/low_mean": 0.000781250006184564, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00450660681235604, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 756.9750366210938, "completions/mean_terminated_length": 559.6087036132812, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.09983661957085133, "epoch": 0.810580204778157, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.017448190599679947, "kl": 0.015745512442663312, "learning_rate": 1.7098976109215017e-06, "loss": 0.0061, "num_tokens": 150357407.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.414259910583496, "sampling/importance_sampling_ratio/mean": 0.3939647078514099, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9497976303100586, "sampling/sampling_logp_difference/mean": 0.012760202400386333, "step": 1425, "step_time": 71.89081695210189 }, { "clip_ratio/high_max": 0.012511824315879494, "clip_ratio/high_mean": 0.0035046910124947317, "clip_ratio/low_mean": 0.0014404297035071068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0049451207160018384, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 888.8500366210938, "completions/mean_terminated_length": 608.1538696289062, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.06717453571036458, "epoch": 0.8111490329920364, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014614447019994259, "kl": 0.011283006810117513, "learning_rate": 1.704778156996587e-06, "loss": -0.0026, "num_tokens": 150465889.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.258551836013794, "sampling/importance_sampling_ratio/mean": 0.21573130786418915, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9782419204711914, "sampling/sampling_logp_difference/mean": 0.01035174261778593, "step": 1426, "step_time": 73.55086914007552 }, { "clip_ratio/high_max": 0.009034738002810627, "clip_ratio/high_mean": 0.0022848334192531183, "clip_ratio/low_mean": 0.001025390622089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003310224012238905, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 981.4000244140625, "completions/mean_terminated_length": 834.6666870117188, "completions/min_length": 455.0, "completions/min_terminated_length": 455.0, "entropy": 0.13445409201085567, "epoch": 0.8117178612059158, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.015947025269269943, "kl": 0.008858762099407613, "learning_rate": 1.6996587030716723e-06, "loss": -0.0962, "num_tokens": 150577633.0, "reward": 0.45000001788139343, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.7396719455718994, "sampling/importance_sampling_ratio/mean": 0.5383202433586121, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4250948429107666, "sampling/sampling_logp_difference/mean": 0.01351807452738285, "step": 1427, "step_time": 77.78179534780793 }, { "clip_ratio/high_max": 0.005301529308781028, "clip_ratio/high_mean": 0.0012012832739856094, "clip_ratio/low_mean": 0.0009952320833690464, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021965153500786982, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.0, "completions/mean_length": 875.2250366210938, "completions/mean_terminated_length": 627.2667236328125, "completions/min_length": 472.0, "completions/min_terminated_length": 472.0, "entropy": 0.10425185412168503, "epoch": 0.8122866894197952, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.03906118869781494, "kl": 0.010368341172579676, "learning_rate": 1.6945392491467576e-06, "loss": -0.1497, "num_tokens": 150684242.0, "reward": 0.699999988079071, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.3060734272003174, "sampling/importance_sampling_ratio/mean": 0.3378373682498932, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0745227336883545, "sampling/sampling_logp_difference/mean": 0.01336930226534605, "step": 1428, "step_time": 73.61152834515087 }, { "clip_ratio/high_max": 0.013191665872000158, "clip_ratio/high_mean": 0.0042807630088645965, "clip_ratio/low_mean": 0.0023606643080711365, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00664142711320892, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 802.0, "completions/mean_length": 752.2250366210938, "completions/mean_terminated_length": 551.3478393554688, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.1474568098783493, "epoch": 0.8128555176336746, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.10050603002309799, "kl": 0.015720531227998435, "learning_rate": 1.689419795221843e-06, "loss": -0.1564, "num_tokens": 150795555.0, "reward": 1.100000023841858, "reward_std": 0.9818872213363647, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.804762363433838, "sampling/importance_sampling_ratio/mean": 0.6017369627952576, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0623347759246826, "sampling/sampling_logp_difference/mean": 0.01727190613746643, "step": 1429, "step_time": 75.31351501541212 }, { "clip_ratio/high_max": 0.012827856466174126, "clip_ratio/high_mean": 0.003675193162052892, "clip_ratio/low_mean": 0.001317869591730414, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004993062757421285, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 899.125, "completions/mean_terminated_length": 711.8125, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.12948859296739101, "epoch": 0.8134243458475541, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.012927043251693249, "kl": 0.012727553956210613, "learning_rate": 1.6843003412969282e-06, "loss": 0.0039, "num_tokens": 150896968.0, "reward": 0.699999988079071, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.408693790435791, "sampling/importance_sampling_ratio/mean": 0.34762707352638245, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4979991912841797, "sampling/sampling_logp_difference/mean": 0.014362278394401073, "step": 1430, "step_time": 74.99077614746056 }, { "clip_ratio/high_max": 0.012350597186014056, "clip_ratio/high_mean": 0.0026456460473127663, "clip_ratio/low_mean": 0.0033935546525754035, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006039200641680509, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 979.4750366210938, "completions/mean_terminated_length": 727.1666870117188, "completions/min_length": 435.0, "completions/min_terminated_length": 435.0, "entropy": 0.14772397745400667, "epoch": 0.8139931740614335, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.010744848288595676, "kl": 0.01272133446764201, "learning_rate": 1.6791808873720137e-06, "loss": 0.0516, "num_tokens": 151008827.0, "reward": 0.2750000059604645, "reward_std": 0.6788942813873291, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 1.7058346271514893, "sampling/importance_sampling_ratio/mean": 0.2731214761734009, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1183245182037354, "sampling/sampling_logp_difference/mean": 0.015130363404750824, "step": 1431, "step_time": 75.10468391794711 }, { "clip_ratio/high_max": 0.012342247529886663, "clip_ratio/high_mean": 0.0033839281531982124, "clip_ratio/low_mean": 0.0016357421736756805, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005019670352339745, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 797.6000366210938, "completions/mean_terminated_length": 571.2000122070312, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.11924672871828079, "epoch": 0.8145620022753128, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02364005520939827, "kl": 0.016292156651616096, "learning_rate": 1.6740614334470989e-06, "loss": 0.0857, "num_tokens": 151112355.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.9211677312850952, "sampling/importance_sampling_ratio/mean": 0.40633219480514526, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.459982395172119, "sampling/sampling_logp_difference/mean": 0.014857261441648006, "step": 1432, "step_time": 80.18340621818788 }, { "clip_ratio/high_max": 0.018582483869977295, "clip_ratio/high_mean": 0.006097037592553534, "clip_ratio/low_mean": 0.0017089843822759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007806021923897788, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 923.0, "completions/mean_length": 789.4750366210938, "completions/mean_terminated_length": 616.1304321289062, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.10978690162301064, "epoch": 0.8151308304891922, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02135816588997841, "kl": 0.010726663516834378, "learning_rate": 1.6689419795221843e-06, "loss": -0.0545, "num_tokens": 151217126.0, "reward": 0.949999988079071, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.5750019550323486, "sampling/importance_sampling_ratio/mean": 0.30558428168296814, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0153727531433105, "sampling/sampling_logp_difference/mean": 0.013833336532115936, "step": 1433, "step_time": 75.45992961642332 }, { "clip_ratio/high_max": 0.013975629233755171, "clip_ratio/high_mean": 0.004210066894302145, "clip_ratio/low_mean": 0.002270507833600277, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006480574782472104, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 938.0, "completions/mean_length": 896.0750122070312, "completions/mean_terminated_length": 658.5, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.10436624940484762, "epoch": 0.8156996587030717, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02787119895219803, "kl": 0.011321722180582583, "learning_rate": 1.6638225255972695e-06, "loss": -0.007, "num_tokens": 151322449.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.98417329788208, "sampling/importance_sampling_ratio/mean": 0.3961697220802307, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6527897119522095, "sampling/sampling_logp_difference/mean": 0.012271369807422161, "step": 1434, "step_time": 76.93230561120436 }, { "clip_ratio/high_max": 0.0134900503908284, "clip_ratio/high_mean": 0.004590656972141005, "clip_ratio/low_mean": 0.00203029240219621, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006620949483476579, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 809.4249877929688, "completions/mean_terminated_length": 594.8500366210938, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.13627608958631754, "epoch": 0.8162684869169511, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018776526674628258, "kl": 0.013878428377211094, "learning_rate": 1.658703071672355e-06, "loss": 0.0037, "num_tokens": 151428498.0, "reward": 0.875, "reward_std": 0.9388344883918762, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.0096685886383057, "sampling/importance_sampling_ratio/mean": 0.27558189630508423, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.136673927307129, "sampling/sampling_logp_difference/mean": 0.015349948778748512, "step": 1435, "step_time": 76.77998442691751 }, { "clip_ratio/high_max": 0.012469428940676153, "clip_ratio/high_mean": 0.002809542347677052, "clip_ratio/low_mean": 0.0016357422100554686, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004445284546818584, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 846.0, "completions/mean_length": 847.1500244140625, "completions/mean_terminated_length": 552.4000244140625, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.09632676932960749, "epoch": 0.8168373151308305, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.012922555208206177, "kl": 0.010196307790465653, "learning_rate": 1.6535836177474402e-06, "loss": 0.0189, "num_tokens": 151531480.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.764585018157959, "sampling/importance_sampling_ratio/mean": 0.3923828899860382, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1426453590393066, "sampling/sampling_logp_difference/mean": 0.012457015924155712, "step": 1436, "step_time": 72.59242933779024 }, { "clip_ratio/high_max": 0.012074961327016354, "clip_ratio/high_mean": 0.002640217440784909, "clip_ratio/low_mean": 0.002252983598737046, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00489320102497004, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 815.0, "completions/mean_length": 745.375, "completions/mean_terminated_length": 559.625, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.10983324982225895, "epoch": 0.8174061433447098, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.030188579112291336, "kl": 0.01891017850721255, "learning_rate": 1.6484641638225256e-06, "loss": -0.0377, "num_tokens": 151639391.0, "reward": 1.1750000715255737, "reward_std": 0.9841695427894592, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.822476863861084, "sampling/importance_sampling_ratio/mean": 0.30598706007003784, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8273308277130127, "sampling/sampling_logp_difference/mean": 0.016526391729712486, "step": 1437, "step_time": 72.33249946055003 }, { "clip_ratio/high_max": 0.01757607504259795, "clip_ratio/high_mean": 0.006718674616422504, "clip_ratio/low_mean": 0.0022674190549878404, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008986093802377582, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 836.7250366210938, "completions/mean_terminated_length": 667.2857055664062, "completions/min_length": 385.0, "completions/min_terminated_length": 385.0, "entropy": 0.14578882046043873, "epoch": 0.8179749715585893, "frac_reward_zero_std": 0.0, "grad_norm": 0.019945325329899788, "kl": 0.013725240365602076, "learning_rate": 1.6433447098976109e-06, "loss": -0.0218, "num_tokens": 151748932.0, "reward": 0.9000000357627869, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.4306524991989136, "sampling/importance_sampling_ratio/mean": 0.1859602928161621, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8139617443084717, "sampling/sampling_logp_difference/mean": 0.016577711328864098, "step": 1438, "step_time": 75.35696974396706 }, { "clip_ratio/high_max": 0.0022870864486321807, "clip_ratio/high_mean": 0.0004574173071887344, "clip_ratio/low_mean": 0.0018554687630967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0023128860630095005, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 894.2250366210938, "completions/mean_terminated_length": 504.8999938964844, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.14448604825884104, "epoch": 0.8185437997724687, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.010723687708377838, "kl": 0.011126227618660778, "learning_rate": 1.6382252559726963e-06, "loss": 0.0227, "num_tokens": 151859741.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.838305950164795, "sampling/importance_sampling_ratio/mean": 0.39068374037742615, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.316732883453369, "sampling/sampling_logp_difference/mean": 0.015483299270272255, "step": 1439, "step_time": 72.18164069694467 }, { "clip_ratio/high_max": 0.015032922674436122, "clip_ratio/high_mean": 0.004105050393263809, "clip_ratio/low_mean": 0.0015138241287786514, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005618874361971393, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 695.7250366210938, "completions/mean_terminated_length": 498.7599792480469, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.0826637796126306, "epoch": 0.8191126279863481, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0561198927462101, "kl": 0.011027640022803098, "learning_rate": 1.6331058020477815e-06, "loss": 0.0742, "num_tokens": 151963106.0, "reward": 1.149999976158142, "reward_std": 0.9486833214759827, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.1894187927246094, "sampling/importance_sampling_ratio/mean": 0.6198599338531494, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8087105751037598, "sampling/sampling_logp_difference/mean": 0.012801336124539375, "step": 1440, "step_time": 72.58237155340612 }, { "clip_ratio/high_max": 0.012718517216853797, "clip_ratio/high_mean": 0.0032041940721683204, "clip_ratio/low_mean": 0.0023437499985448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005547944048885256, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 930.0, "completions/mean_length": 928.625, "completions/mean_terminated_length": 600.1111450195312, "completions/min_length": 424.0, "completions/min_terminated_length": 424.0, "entropy": 0.12102263420820236, "epoch": 0.8196814562002275, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01871834136545658, "kl": 0.011042403755709529, "learning_rate": 1.627986348122867e-06, "loss": 0.036, "num_tokens": 152077779.0, "reward": 0.375, "reward_std": 0.7403221726417542, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.4071710109710693, "sampling/importance_sampling_ratio/mean": 0.32300323247909546, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.596205711364746, "sampling/sampling_logp_difference/mean": 0.01383654959499836, "step": 1441, "step_time": 77.93037568498403 }, { "clip_ratio/high_max": 0.014151613344438374, "clip_ratio/high_mean": 0.0031817787094041705, "clip_ratio/low_mean": 0.0013273540425871033, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00450913276290521, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 808.5250244140625, "completions/mean_terminated_length": 545.1666870117188, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.08034148346632719, "epoch": 0.820250284414107, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02701343409717083, "kl": 0.012501553865149617, "learning_rate": 1.6228668941979522e-06, "loss": 0.0323, "num_tokens": 152185552.0, "reward": 0.8500000238418579, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.8939764499664307, "sampling/importance_sampling_ratio/mean": 0.5974518060684204, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1491613388061523, "sampling/sampling_logp_difference/mean": 0.011797583661973476, "step": 1442, "step_time": 73.51140463608317 }, { "clip_ratio/high_max": 0.013927448526374064, "clip_ratio/high_mean": 0.004492846375796944, "clip_ratio/low_mean": 0.0017822265726863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062750730430707335, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 765.6000366210938, "completions/mean_terminated_length": 574.6087036132812, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.09998280461877584, "epoch": 0.8208191126279863, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04274832829833031, "kl": 0.015796600258909166, "learning_rate": 1.6177474402730376e-06, "loss": 0.077, "num_tokens": 152284656.0, "reward": 1.0500000715255737, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.843690872192383, "sampling/importance_sampling_ratio/mean": 0.56932532787323, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.594890832901001, "sampling/sampling_logp_difference/mean": 0.013666278682649136, "step": 1443, "step_time": 73.62356542609632 }, { "clip_ratio/high_max": 0.014032397069968283, "clip_ratio/high_mean": 0.004620902662281878, "clip_ratio/low_mean": 0.0019420846365392208, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006562987226061523, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 851.0, "completions/mean_length": 817.9750366210938, "completions/mean_terminated_length": 508.9375, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.07397709786891937, "epoch": 0.8213879408418657, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02331089787185192, "kl": 0.012626777752302587, "learning_rate": 1.6126279863481228e-06, "loss": -0.0414, "num_tokens": 152403167.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.0851330757141113, "sampling/importance_sampling_ratio/mean": 0.38971948623657227, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7967066764831543, "sampling/sampling_logp_difference/mean": 0.01059874240309, "step": 1444, "step_time": 76.94005112722516 }, { "clip_ratio/high_max": 0.010543145821429789, "clip_ratio/high_mean": 0.0036233957507647574, "clip_ratio/low_mean": 0.00243625370785594, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0060596496041398495, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 877.4000244140625, "completions/mean_terminated_length": 633.0667114257812, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.122263771481812, "epoch": 0.8219567690557452, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026587406173348427, "kl": 0.011446651653386652, "learning_rate": 1.6075085324232083e-06, "loss": -0.0274, "num_tokens": 152511311.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.4710092544555664, "sampling/importance_sampling_ratio/mean": 0.38327139616012573, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.497448205947876, "sampling/sampling_logp_difference/mean": 0.013691606000065804, "step": 1445, "step_time": 74.82724149199203 }, { "clip_ratio/high_max": 0.015141171868890524, "clip_ratio/high_mean": 0.004350379313109443, "clip_ratio/low_mean": 0.0021972656049911166, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006547645025420934, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 925.9249877929688, "completions/mean_terminated_length": 722.2307739257812, "completions/min_length": 452.0, "completions/min_terminated_length": 452.0, "entropy": 0.15203358326107264, "epoch": 0.8225255972696246, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04191135987639427, "kl": 0.013975810026749969, "learning_rate": 1.6023890784982935e-06, "loss": -0.0856, "num_tokens": 152628396.0, "reward": 0.6000000238418579, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.2350573539733887, "sampling/importance_sampling_ratio/mean": 0.2830275595188141, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3531105518341064, "sampling/sampling_logp_difference/mean": 0.016593346372246742, "step": 1446, "step_time": 77.58725230535492 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 806.8250122070312, "completions/mean_terminated_length": 481.0625, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.08244845643639565, "epoch": 0.823094425483504, "frac_reward_zero_std": 1.0, "grad_norm": 0.001057187095284462, "kl": 0.007804639986716211, "learning_rate": 1.597269624573379e-06, "loss": 0.0001, "num_tokens": 152735405.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.9470906257629395, "sampling/importance_sampling_ratio/mean": 0.6190186142921448, "sampling/importance_sampling_ratio/min": 0.0051302192732691765, "sampling/sampling_logp_difference/max": 1.9265944957733154, "sampling/sampling_logp_difference/mean": 0.01206358801573515, "step": 1447, "step_time": 93.4152715341188 }, { "clip_ratio/high_max": 0.0024146405630744994, "clip_ratio/high_mean": 0.00048292810970451683, "clip_ratio/low_mean": 0.0004150390705035534, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008979671802080702, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 761.1500244140625, "completions/mean_terminated_length": 470.631591796875, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.0872203940525651, "epoch": 0.8236632536973834, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.013612749986350536, "kl": 0.012917145621031523, "learning_rate": 1.5921501706484643e-06, "loss": 0.0524, "num_tokens": 152841323.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.1751887798309326, "sampling/importance_sampling_ratio/mean": 0.3851751387119293, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.003594398498535, "sampling/sampling_logp_difference/mean": 0.01255645789206028, "step": 1448, "step_time": 71.91282610432245 }, { "clip_ratio/high_max": 0.01888151268940419, "clip_ratio/high_mean": 0.00472684929263778, "clip_ratio/low_mean": 0.0012451171896827873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0059719664277508855, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 828.0, "completions/mean_length": 774.125, "completions/mean_terminated_length": 497.9473571777344, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.09641270060092211, "epoch": 0.8242320819112628, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.021224765107035637, "kl": 0.01456733129452914, "learning_rate": 1.5870307167235496e-06, "loss": 0.0202, "num_tokens": 152935448.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.787893772125244, "sampling/importance_sampling_ratio/mean": 0.3367161750793457, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.328369617462158, "sampling/sampling_logp_difference/mean": 0.012350144796073437, "step": 1449, "step_time": 69.47986120730639 }, { "clip_ratio/high_max": 0.009868670080322772, "clip_ratio/high_mean": 0.0038546342693734914, "clip_ratio/low_mean": 0.002554272999987006, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006408907240256667, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 893.0, "completions/mean_length": 918.9000244140625, "completions/mean_terminated_length": 743.7333984375, "completions/min_length": 572.0, "completions/min_terminated_length": 572.0, "entropy": 0.1513744257390499, "epoch": 0.8248009101251422, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.025402361527085304, "kl": 0.02426559606101364, "learning_rate": 1.581911262798635e-06, "loss": -0.0075, "num_tokens": 153049740.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.2609317302703857, "sampling/importance_sampling_ratio/mean": 0.22544991970062256, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.905003547668457, "sampling/sampling_logp_difference/mean": 0.016157377511262894, "step": 1450, "step_time": 77.50943784601986 }, { "clip_ratio/high_max": 0.015518376021645963, "clip_ratio/high_mean": 0.004451747416169383, "clip_ratio/low_mean": 0.0025339404528494924, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006985687825363129, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 940.0, "completions/mean_length": 709.7000122070312, "completions/mean_terminated_length": 540.4615478515625, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.1079368507489562, "epoch": 0.8253697383390216, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03528166562318802, "kl": 0.014616891625337303, "learning_rate": 1.5767918088737202e-06, "loss": -0.1124, "num_tokens": 153152392.0, "reward": 1.149999976158142, "reward_std": 0.9212589859962463, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.3804380893707275, "sampling/importance_sampling_ratio/mean": 0.5710598230361938, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.825068473815918, "sampling/sampling_logp_difference/mean": 0.014657141640782356, "step": 1451, "step_time": 71.46239552111365 }, { "clip_ratio/high_max": 0.012300069793127477, "clip_ratio/high_mean": 0.004289786680601537, "clip_ratio/low_mean": 0.00297851564391749, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007268302200827748, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 959.0, "completions/mean_length": 893.6000366210938, "completions/mean_terminated_length": 734.2222290039062, "completions/min_length": 479.0, "completions/min_terminated_length": 479.0, "entropy": 0.1390589289367199, "epoch": 0.825938566552901, "frac_reward_zero_std": 0.0, "grad_norm": 0.016469918191432953, "kl": 0.011486878618597984, "learning_rate": 1.5716723549488057e-06, "loss": 0.0082, "num_tokens": 153270024.0, "reward": 0.7250000238418579, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.370187759399414, "sampling/importance_sampling_ratio/mean": 0.2894880771636963, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.034086227416992, "sampling/sampling_logp_difference/mean": 0.015015971846878529, "step": 1452, "step_time": 82.30474236304872 }, { "clip_ratio/high_max": 0.014099864580202848, "clip_ratio/high_mean": 0.004479063703911379, "clip_ratio/low_mean": 0.001269531247089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005748594790929928, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 657.4500122070312, "completions/mean_terminated_length": 565.8125, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.13041708152741194, "epoch": 0.8265073947667805, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015112637542188168, "kl": 0.014643535949289799, "learning_rate": 1.5665529010238909e-06, "loss": -0.0137, "num_tokens": 153376546.0, "reward": 1.5750000476837158, "reward_std": 0.8129559755325317, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.7749999761581421, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.2580997943878174, "sampling/importance_sampling_ratio/mean": 0.4189896583557129, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.039555788040161, "sampling/sampling_logp_difference/mean": 0.01639210246503353, "step": 1453, "step_time": 72.88519720407203 }, { "clip_ratio/high_max": 0.009755992679856718, "clip_ratio/high_mean": 0.002791761864500586, "clip_ratio/low_mean": 0.0010742187732830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038659805577481166, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 807.0, "completions/mean_length": 824.2250366210938, "completions/mean_terminated_length": 580.0555419921875, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "entropy": 0.08619097899645567, "epoch": 0.8270762229806599, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022623399272561073, "kl": 0.009136296343058348, "learning_rate": 1.5614334470989763e-06, "loss": 0.076, "num_tokens": 153470019.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.1821346282958984, "sampling/importance_sampling_ratio/mean": 0.4308586120605469, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8641855716705322, "sampling/sampling_logp_difference/mean": 0.010860827751457691, "step": 1454, "step_time": 71.49098533531651 }, { "clip_ratio/high_max": 0.017732644802890718, "clip_ratio/high_mean": 0.00681130267912522, "clip_ratio/low_mean": 0.0024727732125029434, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009284076106268913, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 865.0, "completions/mean_length": 837.6749877929688, "completions/mean_terminated_length": 651.3500366210938, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.12817727774381638, "epoch": 0.8276450511945392, "frac_reward_zero_std": 0.0, "grad_norm": 0.031079404056072235, "kl": 0.012996602454222739, "learning_rate": 1.5563139931740615e-06, "loss": 0.0048, "num_tokens": 153581086.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9131851196289062, "sampling/importance_sampling_ratio/mean": 0.6669252514839172, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2589290142059326, "sampling/sampling_logp_difference/mean": 0.014628355391323566, "step": 1455, "step_time": 78.05175396404229 }, { "clip_ratio/high_max": 0.010350346623454243, "clip_ratio/high_mean": 0.0026635415852069855, "clip_ratio/low_mean": 0.0017843991445261054, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004447940620593727, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.0, "completions/mean_length": 931.625, "completions/mean_terminated_length": 688.0909423828125, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.08151919487863779, "epoch": 0.8282138794084186, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014495081268250942, "kl": 0.013037714816164225, "learning_rate": 1.551194539249147e-06, "loss": 0.0693, "num_tokens": 153690191.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.6040890216827393, "sampling/importance_sampling_ratio/mean": 0.33465033769607544, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3247599601745605, "sampling/sampling_logp_difference/mean": 0.011366898193955421, "step": 1456, "step_time": 75.38099012500606 }, { "clip_ratio/high_max": 0.006687164481263608, "clip_ratio/high_mean": 0.001337432891887147, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032905578846111894, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 976.625, "completions/mean_terminated_length": 708.1666870117188, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.09264297224581242, "epoch": 0.8287827076222981, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014009523205459118, "kl": 0.01230379065964371, "learning_rate": 1.5460750853242322e-06, "loss": -0.0233, "num_tokens": 153796096.0, "reward": 0.30000001192092896, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.15000000596046448, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 1.4247864484786987, "sampling/importance_sampling_ratio/mean": 0.27052584290504456, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.5931735038757324, "sampling/sampling_logp_difference/mean": 0.011237788945436478, "step": 1457, "step_time": 72.31624390510842 }, { "clip_ratio/high_max": 0.009757554740644991, "clip_ratio/high_mean": 0.0029866558325011283, "clip_ratio/low_mean": 0.0020019531075377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004988608983694576, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 859.5250244140625, "completions/mean_terminated_length": 658.5, "completions/min_length": 452.0, "completions/min_terminated_length": 452.0, "entropy": 0.1296746889129281, "epoch": 0.8293515358361775, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019219722598791122, "kl": 0.01022849598666653, "learning_rate": 1.5409556313993176e-06, "loss": -0.0501, "num_tokens": 153906253.0, "reward": 0.875, "reward_std": 0.9919548630714417, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.4547924995422363, "sampling/importance_sampling_ratio/mean": 0.46733084321022034, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.879671573638916, "sampling/sampling_logp_difference/mean": 0.01405345369130373, "step": 1458, "step_time": 76.71703549614176 }, { "clip_ratio/high_max": 0.011874527554027736, "clip_ratio/high_mean": 0.0037040587194496766, "clip_ratio/low_mean": 0.001220703143189894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004924761829897761, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 945.0, "completions/mean_length": 887.5, "completions/mean_terminated_length": 660.0000610351562, "completions/min_length": 439.0, "completions/min_terminated_length": 439.0, "entropy": 0.07632262911647558, "epoch": 0.8299203640500569, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01876327395439148, "kl": 0.00852945283986628, "learning_rate": 1.5358361774744028e-06, "loss": -0.073, "num_tokens": 154005657.0, "reward": 0.6000000238418579, "reward_std": 0.8412445187568665, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.7148983478546143, "sampling/importance_sampling_ratio/mean": 0.42404428124427795, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.598088264465332, "sampling/sampling_logp_difference/mean": 0.01069250050932169, "step": 1459, "step_time": 72.83672963595018 }, { "clip_ratio/high_max": 0.012913001235574484, "clip_ratio/high_mean": 0.004419037140905857, "clip_ratio/low_mean": 0.002199193850174197, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006618230836465955, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 1024.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 629.5750122070312, "completions/mean_terminated_length": 559.9705810546875, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.10784605517983437, "epoch": 0.8304891922639362, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.016489753499627113, "kl": 0.012731441645883024, "learning_rate": 1.5307167235494883e-06, "loss": -0.0102, "num_tokens": 154101232.0, "reward": 1.4500000476837158, "reward_std": 0.7493587136268616, "rewards/format_reward/mean": 0.8500000238418579, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.5884130001068115, "sampling/importance_sampling_ratio/mean": 0.3155421316623688, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7973761558532715, "sampling/sampling_logp_difference/mean": 0.014927811920642853, "step": 1460, "step_time": 73.38696620916016 }, { "clip_ratio/high_max": 0.003733151825144887, "clip_ratio/high_mean": 0.0008122471626847982, "clip_ratio/low_mean": 0.0015869140552240424, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0023991612033569254, "completions/clipped_ratio": 0.925000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 923.0, "completions/mean_length": 1000.0, "completions/mean_terminated_length": 704.0, "completions/min_length": 381.0, "completions/min_terminated_length": 381.0, "entropy": 0.12192773818969727, "epoch": 0.8310580204778157, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.004935941658914089, "kl": 0.007762092223856598, "learning_rate": 1.5255972696245733e-06, "loss": 0.0378, "num_tokens": 154216824.0, "reward": 0.10000000149011612, "reward_std": 0.378932386636734, "rewards/format_reward/mean": 0.07500000298023224, "rewards/format_reward/std": 0.26674678921699524, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.1464293003082275, "sampling/importance_sampling_ratio/mean": 0.40408429503440857, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7507752180099487, "sampling/sampling_logp_difference/mean": 0.01230615470558405, "step": 1461, "step_time": 77.43262945022434 }, { "clip_ratio/high_max": 0.01744613057235256, "clip_ratio/high_mean": 0.006541339389514178, "clip_ratio/low_mean": 0.0017333984578726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008274737861938775, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 960.0, "completions/mean_length": 752.9249877929688, "completions/mean_terminated_length": 481.8500061035156, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.12852112762629986, "epoch": 0.8316268486916951, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015204014256596565, "kl": 0.01482327631674707, "learning_rate": 1.5204778156996587e-06, "loss": 0.0415, "num_tokens": 154322109.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.7220300436019897, "sampling/importance_sampling_ratio/mean": 0.273847758769989, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9580278396606445, "sampling/sampling_logp_difference/mean": 0.014843672513961792, "step": 1462, "step_time": 75.87166678160429 }, { "clip_ratio/high_max": 0.005245737571385689, "clip_ratio/high_mean": 0.0012936462881043553, "clip_ratio/low_mean": 0.0011474609491415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024411072226939723, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 828.7000122070312, "completions/mean_terminated_length": 466.0000305175781, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.15395990759134293, "epoch": 0.8321956769055745, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.0074345762841403484, "kl": 0.012017407920211554, "learning_rate": 1.515358361774744e-06, "loss": -0.0117, "num_tokens": 154429097.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.0837470293045044, "sampling/importance_sampling_ratio/mean": 0.24501773715019226, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8761847019195557, "sampling/sampling_logp_difference/mean": 0.017039906233549118, "step": 1463, "step_time": 74.93548249988817 }, { "clip_ratio/high_max": 0.012307077704463154, "clip_ratio/high_mean": 0.004790138831594959, "clip_ratio/low_mean": 0.002246093783469405, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007036232564132661, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 961.0, "completions/mean_length": 827.3250122070312, "completions/mean_terminated_length": 630.6500244140625, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.1265908284112811, "epoch": 0.8327645051194539, "frac_reward_zero_std": 0.0, "grad_norm": 0.02950097620487213, "kl": 0.010694221360608935, "learning_rate": 1.5102389078498294e-06, "loss": -0.0403, "num_tokens": 154531214.0, "reward": 0.925000011920929, "reward_std": 0.9710555076599121, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.2371158599853516, "sampling/importance_sampling_ratio/mean": 0.4068354070186615, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7742667198181152, "sampling/sampling_logp_difference/mean": 0.014192377217113972, "step": 1464, "step_time": 72.57746795332059 }, { "clip_ratio/high_max": 0.010998738638591021, "clip_ratio/high_mean": 0.0031524856749456376, "clip_ratio/low_mean": 0.0025957941616070457, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005748279858380556, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 930.0, "completions/mean_length": 817.6749877929688, "completions/mean_terminated_length": 565.5, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.09230081643909216, "epoch": 0.8333333333333334, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.025668755173683167, "kl": 0.01234168908558786, "learning_rate": 1.5051194539249146e-06, "loss": 0.0323, "num_tokens": 154624689.0, "reward": 0.625, "reward_std": 0.7741827964782715, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.226961135864258, "sampling/importance_sampling_ratio/mean": 0.42541083693504333, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.380462646484375, "sampling/sampling_logp_difference/mean": 0.012143491767346859, "step": 1465, "step_time": 71.19730573892593 }, { "clip_ratio/high_max": 0.010907688119914383, "clip_ratio/high_mean": 0.0024579204109613784, "clip_ratio/low_mean": 0.001708984375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004166904865996912, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 719.125, "completions/mean_terminated_length": 536.2000122070312, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.12681268155574799, "epoch": 0.8339021615472128, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014574408531188965, "kl": 0.013601879472844303, "learning_rate": 1.5e-06, "loss": 0.037, "num_tokens": 154717118.0, "reward": 1.25, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.158496618270874, "sampling/importance_sampling_ratio/mean": 0.4392010569572449, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6191787719726562, "sampling/sampling_logp_difference/mean": 0.015310960821807384, "step": 1466, "step_time": 78.12828859407455 }, { "clip_ratio/high_max": 0.009247574838809669, "clip_ratio/high_mean": 0.002206692828622181, "clip_ratio/low_mean": 0.0016917795655899681, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038984724087640643, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 881.7000122070312, "completions/mean_terminated_length": 644.5333862304688, "completions/min_length": 463.0, "completions/min_terminated_length": 463.0, "entropy": 0.08751827385276556, "epoch": 0.8344709897610921, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014861371368169785, "kl": 0.008980744518339634, "learning_rate": 1.4948805460750853e-06, "loss": -0.0004, "num_tokens": 154819154.0, "reward": 0.625, "reward_std": 0.8678738474845886, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.5216760635375977, "sampling/importance_sampling_ratio/mean": 0.3563566505908966, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.202122211456299, "sampling/sampling_logp_difference/mean": 0.011340809054672718, "step": 1467, "step_time": 76.88492044922896 }, { "clip_ratio/high_max": 0.012161255232058465, "clip_ratio/high_mean": 0.0032664342506905086, "clip_ratio/low_mean": 0.0010958059574477375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0043622403318295255, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 703.0499877929688, "completions/mean_terminated_length": 596.0667114257812, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.08833027817308903, "epoch": 0.8350398179749715, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01421458087861538, "kl": 0.011815558653324842, "learning_rate": 1.4897610921501707e-06, "loss": 0.0408, "num_tokens": 154916220.0, "reward": 1.4250000715255737, "reward_std": 0.8737629055976868, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.6649842262268066, "sampling/importance_sampling_ratio/mean": 0.4205505847930908, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5947842597961426, "sampling/sampling_logp_difference/mean": 0.012018966488540173, "step": 1468, "step_time": 74.06265313155018 }, { "clip_ratio/high_max": 0.00883065618108958, "clip_ratio/high_mean": 0.0029012090526521206, "clip_ratio/low_mean": 0.0017333984505967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004634607466869056, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 895.6000366210938, "completions/mean_terminated_length": 703.0, "completions/min_length": 486.0, "completions/min_terminated_length": 486.0, "entropy": 0.07846223097294569, "epoch": 0.835608646188851, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030596673488616943, "kl": 0.011521732900291681, "learning_rate": 1.484641638225256e-06, "loss": -0.0842, "num_tokens": 155022676.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.45070219039917, "sampling/importance_sampling_ratio/mean": 0.31823086738586426, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0440685749053955, "sampling/sampling_logp_difference/mean": 0.010833051055669785, "step": 1469, "step_time": 73.50743054365739 }, { "clip_ratio/high_max": 0.015131000429391861, "clip_ratio/high_mean": 0.0064737541251815856, "clip_ratio/low_mean": 0.002026367190410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008500121184624732, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 843.5250244140625, "completions/mean_terminated_length": 710.1304321289062, "completions/min_length": 434.0, "completions/min_terminated_length": 434.0, "entropy": 0.1443699412047863, "epoch": 0.8361774744027304, "frac_reward_zero_std": 0.0, "grad_norm": 0.02236981876194477, "kl": 0.012672970187850296, "learning_rate": 1.4795221843003414e-06, "loss": -0.1369, "num_tokens": 155130057.0, "reward": 1.024999976158142, "reward_std": 0.9469926357269287, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.823103427886963, "sampling/importance_sampling_ratio/mean": 0.3523311913013458, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.749293804168701, "sampling/sampling_logp_difference/mean": 0.015690341591835022, "step": 1470, "step_time": 73.6382657168433 }, { "clip_ratio/high_max": 0.011506830574944615, "clip_ratio/high_mean": 0.0029632279474753886, "clip_ratio/low_mean": 0.0016601562310825102, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004623384156730026, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 872.0, "completions/mean_length": 867.9000244140625, "completions/mean_terminated_length": 677.1111450195312, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "entropy": 0.12201043590903282, "epoch": 0.8367463026166098, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018930580466985703, "kl": 0.012036223779432476, "learning_rate": 1.4744027303754266e-06, "loss": -0.0337, "num_tokens": 155241973.0, "reward": 0.9000000357627869, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.7317159175872803, "sampling/importance_sampling_ratio/mean": 0.3631516098976135, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.832634210586548, "sampling/sampling_logp_difference/mean": 0.01471645850688219, "step": 1471, "step_time": 76.71725805941969 }, { "clip_ratio/high_max": 0.008073456934653223, "clip_ratio/high_mean": 0.0022941180286579765, "clip_ratio/low_mean": 0.001049804681315436, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003343922711792402, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 846.5, "completions/mean_terminated_length": 550.6666870117188, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.10303772054612637, "epoch": 0.8373151308304891, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02273828163743019, "kl": 0.01302230980945751, "learning_rate": 1.469283276450512e-06, "loss": 0.0221, "num_tokens": 155347641.0, "reward": 0.6000000238418579, "reward_std": 0.8412445187568665, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.477243423461914, "sampling/importance_sampling_ratio/mean": 0.4156733453273773, "sampling/importance_sampling_ratio/min": 0.009740849956870079, "sampling/sampling_logp_difference/max": 1.9321022033691406, "sampling/sampling_logp_difference/mean": 0.012088407762348652, "step": 1472, "step_time": 77.25431276881136 }, { "clip_ratio/high_max": 0.010225160280242562, "clip_ratio/high_mean": 0.0034846719063352793, "clip_ratio/low_mean": 0.0011718750065483619, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004656546894693747, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 819.0750122070312, "completions/mean_terminated_length": 651.4091186523438, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.09190270118415356, "epoch": 0.8378839590443686, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01193125732243061, "kl": 0.012605962925590575, "learning_rate": 1.4641638225255972e-06, "loss": 0.0134, "num_tokens": 155448476.0, "reward": 0.7750000357627869, "reward_std": 0.8002403378486633, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.6963322162628174, "sampling/importance_sampling_ratio/mean": 0.3952093720436096, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1797237396240234, "sampling/sampling_logp_difference/mean": 0.011937123723328114, "step": 1473, "step_time": 72.95820403727703 }, { "clip_ratio/high_max": 0.011614722665399313, "clip_ratio/high_mean": 0.0029201113065937534, "clip_ratio/low_mean": 0.0003906250076397555, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003310736305138562, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 812.2250366210938, "completions/mean_terminated_length": 553.388916015625, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.10471259243786335, "epoch": 0.838452787258248, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.007621066179126501, "kl": 0.011461456073448062, "learning_rate": 1.4590443686006827e-06, "loss": -0.0184, "num_tokens": 155558469.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.9943971633911133, "sampling/importance_sampling_ratio/mean": 0.3527801036834717, "sampling/importance_sampling_ratio/min": 0.00036855379585176706, "sampling/sampling_logp_difference/max": 1.9742237329483032, "sampling/sampling_logp_difference/mean": 0.013854125514626503, "step": 1474, "step_time": 75.22374035045505 }, { "clip_ratio/high_max": 0.012920151697471738, "clip_ratio/high_mean": 0.004701765486970544, "clip_ratio/low_mean": 0.0021014379108237335, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006803203199524432, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 967.0, "completions/mean_length": 844.9500122070312, "completions/mean_terminated_length": 647.0526123046875, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.11875302903354168, "epoch": 0.8390216154721274, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03710139915347099, "kl": 0.014308675890788436, "learning_rate": 1.4539249146757679e-06, "loss": -0.0899, "num_tokens": 155668003.0, "reward": 0.9000000357627869, "reward_std": 0.9818872213363647, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.288233757019043, "sampling/importance_sampling_ratio/mean": 0.4683896601200104, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.907352924346924, "sampling/sampling_logp_difference/mean": 0.013959958218038082, "step": 1475, "step_time": 73.9613537513651 }, { "clip_ratio/high_max": 0.01352081773802638, "clip_ratio/high_mean": 0.0035360026522539556, "clip_ratio/low_mean": 0.0010009765574068297, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004536979191470891, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 843.0, "completions/mean_length": 780.7750244140625, "completions/mean_terminated_length": 560.7142944335938, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.07293745595961809, "epoch": 0.8395904436860068, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.021499594673514366, "kl": 0.010595438070595264, "learning_rate": 1.4488054607508533e-06, "loss": -0.0394, "num_tokens": 155766394.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.6439530849456787, "sampling/importance_sampling_ratio/mean": 0.5340186357498169, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7056422233581543, "sampling/sampling_logp_difference/mean": 0.010712987743318081, "step": 1476, "step_time": 72.79927573003806 }, { "clip_ratio/high_max": 0.015668842068407685, "clip_ratio/high_mean": 0.006295830346061848, "clip_ratio/low_mean": 0.0017333984360448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008029228774830699, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.0, "completions/mean_length": 765.7999877929688, "completions/mean_terminated_length": 507.6000061035156, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.09893102571368217, "epoch": 0.8401592718998863, "frac_reward_zero_std": 0.0, "grad_norm": 0.031707897782325745, "kl": 0.013876995421014726, "learning_rate": 1.4436860068259385e-06, "loss": 0.003, "num_tokens": 155872290.0, "reward": 1.0, "reward_std": 1.0127393007278442, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.7278034687042236, "sampling/importance_sampling_ratio/mean": 0.4363884925842285, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3832249641418457, "sampling/sampling_logp_difference/mean": 0.01302648801356554, "step": 1477, "step_time": 75.30605429667048 }, { "clip_ratio/high_max": 0.016680256056133658, "clip_ratio/high_mean": 0.007254471689520869, "clip_ratio/low_mean": 0.0009702126626507379, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008224684279412031, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 802.4500122070312, "completions/mean_terminated_length": 683.1538696289062, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "entropy": 0.10433985944837332, "epoch": 0.8407281001137656, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014780902303755283, "kl": 0.012241609510965645, "learning_rate": 1.438566552901024e-06, "loss": -0.0589, "num_tokens": 155966836.0, "reward": 1.274999976158142, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.8812239170074463, "sampling/importance_sampling_ratio/mean": 0.3903600573539734, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0277700424194336, "sampling/sampling_logp_difference/mean": 0.013484815135598183, "step": 1478, "step_time": 70.8919285130687 }, { "clip_ratio/high_max": 0.013119828188791871, "clip_ratio/high_mean": 0.004146293067606166, "clip_ratio/low_mean": 0.0017578124825377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005904105521040037, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 868.5750122070312, "completions/mean_terminated_length": 579.9285888671875, "completions/min_length": 396.0, "completions/min_terminated_length": 396.0, "entropy": 0.09242717223241925, "epoch": 0.841296928327645, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.019565774127840996, "kl": 0.009539517166558653, "learning_rate": 1.4334470989761092e-06, "loss": 0.0619, "num_tokens": 156066323.0, "reward": 0.699999988079071, "reward_std": 0.9660917520523071, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.728149175643921, "sampling/importance_sampling_ratio/mean": 0.3795783519744873, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4599812030792236, "sampling/sampling_logp_difference/mean": 0.011460807174444199, "step": 1479, "step_time": 71.54221171280369 }, { "clip_ratio/high_max": 0.012143268424551934, "clip_ratio/high_mean": 0.003455206679063849, "clip_ratio/low_mean": 0.0021322451138985343, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005587451858446002, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 860.7000122070312, "completions/mean_terminated_length": 661.1111450195312, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.1481089647859335, "epoch": 0.8418657565415245, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024209022521972656, "kl": 0.013079783588182181, "learning_rate": 1.4283276450511946e-06, "loss": -0.094, "num_tokens": 156178319.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.834038019180298, "sampling/importance_sampling_ratio/mean": 0.45213884115219116, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.80106258392334, "sampling/sampling_logp_difference/mean": 0.015078321099281311, "step": 1480, "step_time": 76.83143774420023 }, { "clip_ratio/high_max": 0.010926353803370148, "clip_ratio/high_mean": 0.0028991292201681063, "clip_ratio/low_mean": 0.0012451171896827873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004144246457144618, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 802.375, "completions/mean_terminated_length": 580.75, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.07407884579151869, "epoch": 0.8424345847554039, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03193936124444008, "kl": 0.01074393087765202, "learning_rate": 1.4232081911262799e-06, "loss": 0.0089, "num_tokens": 156270062.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.9922235012054443, "sampling/importance_sampling_ratio/mean": 0.4093237817287445, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9412269592285156, "sampling/sampling_logp_difference/mean": 0.011088674888014793, "step": 1481, "step_time": 72.1578113459982 }, { "clip_ratio/high_max": 0.010181641322560608, "clip_ratio/high_mean": 0.0036256600287742913, "clip_ratio/low_mean": 0.0023975257499841973, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006023185793310404, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 709.7000122070312, "completions/mean_terminated_length": 521.1199951171875, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.11231181025505066, "epoch": 0.8430034129692833, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02803811989724636, "kl": 0.013523673405870795, "learning_rate": 1.4180887372013653e-06, "loss": 0.1617, "num_tokens": 156369466.0, "reward": 1.0750000476837158, "reward_std": 0.9167249798774719, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9830706119537354, "sampling/importance_sampling_ratio/mean": 0.551963210105896, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0866267681121826, "sampling/sampling_logp_difference/mean": 0.01446321327239275, "step": 1482, "step_time": 69.91805986571126 }, { "clip_ratio/high_max": 0.011774568818509579, "clip_ratio/high_mean": 0.0032656827606842853, "clip_ratio/low_mean": 0.0024013837974052876, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005667066492605954, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 759.8500366210938, "completions/mean_terminated_length": 583.75, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.0708316033706069, "epoch": 0.8435722411831627, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03929390758275986, "kl": 0.009786174166947603, "learning_rate": 1.4129692832764505e-06, "loss": 0.0024, "num_tokens": 156470220.0, "reward": 0.9000000357627869, "reward_std": 0.8412445187568665, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.907266616821289, "sampling/importance_sampling_ratio/mean": 0.6023674011230469, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3553249835968018, "sampling/sampling_logp_difference/mean": 0.010787545703351498, "step": 1483, "step_time": 75.78639574348927 }, { "clip_ratio/high_max": 0.009993558749556541, "clip_ratio/high_mean": 0.0029510233434848487, "clip_ratio/low_mean": 0.0009521484498691279, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0039031717169564217, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 679.2250366210938, "completions/mean_terminated_length": 564.300048828125, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.08357803151011467, "epoch": 0.8441410693970421, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019410086795687675, "kl": 0.018727459013462067, "learning_rate": 1.407849829351536e-06, "loss": -0.059, "num_tokens": 156563981.0, "reward": 1.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.75, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.9520939588546753, "sampling/importance_sampling_ratio/mean": 0.33039188385009766, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5948920249938965, "sampling/sampling_logp_difference/mean": 0.013524800539016724, "step": 1484, "step_time": 72.02335221413523 }, { "clip_ratio/high_max": 0.01369779696688056, "clip_ratio/high_mean": 0.00606328877620399, "clip_ratio/low_mean": 0.001861199998529628, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.00792448886204511, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 934.0, "completions/mean_length": 752.4000244140625, "completions/mean_terminated_length": 589.4400024414062, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.11052969098091125, "epoch": 0.8447098976109215, "frac_reward_zero_std": 0.0, "grad_norm": 0.028438040986657143, "kl": 0.01436778623610735, "learning_rate": 1.4027303754266212e-06, "loss": -0.0403, "num_tokens": 156672525.0, "reward": 1.0500000715255737, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.8354787826538086, "sampling/importance_sampling_ratio/mean": 0.35188981890678406, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.6684703826904297, "sampling/sampling_logp_difference/mean": 0.013956702314317226, "step": 1485, "step_time": 76.98238475504331 }, { "clip_ratio/high_max": 0.014361813198775053, "clip_ratio/high_mean": 0.005712718004360795, "clip_ratio/low_mean": 0.0020840372671955265, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007796755118761212, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 925.0, "completions/mean_length": 734.4249877929688, "completions/mean_terminated_length": 497.5, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.0805899165570736, "epoch": 0.8452787258248009, "frac_reward_zero_std": 0.0, "grad_norm": 0.02990332990884781, "kl": 0.013669450883753598, "learning_rate": 1.3976109215017066e-06, "loss": -0.0104, "num_tokens": 156775502.0, "reward": 1.024999976158142, "reward_std": 0.9736924767494202, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.3471715450286865, "sampling/importance_sampling_ratio/mean": 0.4802533984184265, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.387720823287964, "sampling/sampling_logp_difference/mean": 0.01176383625715971, "step": 1486, "step_time": 75.92056013317779 }, { "clip_ratio/high_max": 0.014485681196674705, "clip_ratio/high_mean": 0.004440125761902891, "clip_ratio/low_mean": 0.0013592152263299795, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00579934089910239, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 813.0, "completions/mean_length": 824.7000122070312, "completions/mean_terminated_length": 525.75, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.0730297127738595, "epoch": 0.8458475540386803, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01745162159204483, "kl": 0.007865982363000512, "learning_rate": 1.3924914675767918e-06, "loss": 0.0504, "num_tokens": 156862642.0, "reward": 0.75, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.3938772678375244, "sampling/importance_sampling_ratio/mean": 0.5150625705718994, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8622422218322754, "sampling/sampling_logp_difference/mean": 0.010649326257407665, "step": 1487, "step_time": 71.58011318091303 }, { "clip_ratio/high_max": 0.006827085744589567, "clip_ratio/high_mean": 0.0013654171489179134, "clip_ratio/low_mean": 0.0014160156206344254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0027814327550004236, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 813.0, "completions/mean_length": 833.375, "completions/mean_terminated_length": 515.6666870117188, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.12047028448432684, "epoch": 0.8464163822525598, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.013017104007303715, "kl": 0.01327280595432967, "learning_rate": 1.3873720136518773e-06, "loss": -0.026, "num_tokens": 156973337.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.4769538640975952, "sampling/importance_sampling_ratio/mean": 0.391505628824234, "sampling/importance_sampling_ratio/min": 0.005439485423266888, "sampling/sampling_logp_difference/max": 1.988077163696289, "sampling/sampling_logp_difference/mean": 0.01381008792668581, "step": 1488, "step_time": 74.36238451232202 }, { "clip_ratio/high_max": 0.014936852036044002, "clip_ratio/high_mean": 0.0038694872346241027, "clip_ratio/low_mean": 0.002343749991268851, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006213237211341038, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 866.0, "completions/mean_length": 916.6749877929688, "completions/mean_terminated_length": 633.727294921875, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.10093120113015175, "epoch": 0.8469852104664392, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.039172857999801636, "kl": 0.01015585649292916, "learning_rate": 1.3822525597269625e-06, "loss": -0.0129, "num_tokens": 157085644.0, "reward": 0.550000011920929, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.277660846710205, "sampling/importance_sampling_ratio/mean": 0.47679299116134644, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.910202980041504, "sampling/sampling_logp_difference/mean": 0.011914088390767574, "step": 1489, "step_time": 74.06642780010588 }, { "clip_ratio/high_max": 0.010302169364877045, "clip_ratio/high_mean": 0.0027052955265389755, "clip_ratio/low_mean": 0.0025634765916038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005268772220006213, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 847.0, "completions/mean_length": 937.8500366210938, "completions/mean_terminated_length": 593.25, "completions/min_length": 387.0, "completions/min_terminated_length": 387.0, "entropy": 0.11109972279518843, "epoch": 0.8475540386803185, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.07272788882255554, "kl": 0.010238584014587104, "learning_rate": 1.377133105802048e-06, "loss": -0.1202, "num_tokens": 157194990.0, "reward": 0.4000000059604645, "reward_std": 0.8101915121078491, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.770479440689087, "sampling/importance_sampling_ratio/mean": 0.39479684829711914, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9386038780212402, "sampling/sampling_logp_difference/mean": 0.012226332910358906, "step": 1490, "step_time": 74.51368649466895 }, { "clip_ratio/high_max": 0.011254598444793373, "clip_ratio/high_mean": 0.002626668123411946, "clip_ratio/low_mean": 0.002046551562671084, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004673219635151327, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 925.5250244140625, "completions/mean_terminated_length": 665.9091186523438, "completions/min_length": 448.0, "completions/min_terminated_length": 448.0, "entropy": 0.12331814505159855, "epoch": 0.8481228668941979, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02488658018410206, "kl": 0.01282428193371743, "learning_rate": 1.3720136518771331e-06, "loss": 0.01, "num_tokens": 157308803.0, "reward": 0.5250000357627869, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.7921478748321533, "sampling/importance_sampling_ratio/mean": 0.4910239577293396, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1796975135803223, "sampling/sampling_logp_difference/mean": 0.013849637471139431, "step": 1491, "step_time": 75.21440485678613 }, { "clip_ratio/high_max": 0.01083630247740075, "clip_ratio/high_mean": 0.0029654129029950127, "clip_ratio/low_mean": 0.003664256539195776, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006629669514950365, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 829.25, "completions/mean_terminated_length": 537.125, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.1223162841051817, "epoch": 0.8486916951080774, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.011784021742641926, "kl": 0.012606975389644504, "learning_rate": 1.3668941979522186e-06, "loss": 0.1152, "num_tokens": 157412781.0, "reward": 0.675000011920929, "reward_std": 0.8883144855499268, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.7249460220336914, "sampling/importance_sampling_ratio/mean": 0.43686699867248535, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.118206262588501, "sampling/sampling_logp_difference/mean": 0.014186953194439411, "step": 1492, "step_time": 75.44806409906596 }, { "clip_ratio/high_max": 0.011581436381675303, "clip_ratio/high_mean": 0.0037481393082998693, "clip_ratio/low_mean": 0.002026367190410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005774506484158337, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 967.0, "completions/mean_length": 826.5499877929688, "completions/mean_terminated_length": 608.3157958984375, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.0946623869240284, "epoch": 0.8492605233219568, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03586201369762421, "kl": 0.012381483218632638, "learning_rate": 1.361774744027304e-06, "loss": -0.0312, "num_tokens": 157515251.0, "reward": 0.949999988079071, "reward_std": 1.0114725828170776, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.756834030151367, "sampling/importance_sampling_ratio/mean": 0.38903072476387024, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6645431518554688, "sampling/sampling_logp_difference/mean": 0.012131543830037117, "step": 1493, "step_time": 74.30888566374779 }, { "clip_ratio/high_max": 0.014846083475276828, "clip_ratio/high_mean": 0.005000258708605543, "clip_ratio/low_mean": 0.0015625000232830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006562758644577116, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 865.6000366210938, "completions/mean_terminated_length": 651.2941284179688, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "entropy": 0.10213720425963402, "epoch": 0.8498293515358362, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.031238647177815437, "kl": 0.011649612803012133, "learning_rate": 1.3566552901023892e-06, "loss": -0.0011, "num_tokens": 157629531.0, "reward": 0.699999988079071, "reward_std": 0.8828856945037842, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.901895046234131, "sampling/importance_sampling_ratio/mean": 0.44234925508499146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.039999485015869, "sampling/sampling_logp_difference/mean": 0.01331714540719986, "step": 1494, "step_time": 75.8645834357012 }, { "clip_ratio/high_max": 0.019214341999031603, "clip_ratio/high_mean": 0.006298372813034803, "clip_ratio/low_mean": 0.0007324218640860636, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007030794775346294, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 944.0, "completions/mean_length": 876.2999877929688, "completions/mean_terminated_length": 676.4705810546875, "completions/min_length": 406.0, "completions/min_terminated_length": 406.0, "entropy": 0.12914741225540638, "epoch": 0.8503981797497155, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.05550820007920265, "kl": 0.01003861939534545, "learning_rate": 1.3515358361774747e-06, "loss": 0.2127, "num_tokens": 157736487.0, "reward": 0.675000011920929, "reward_std": 0.888314425945282, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.1355037689208984, "sampling/importance_sampling_ratio/mean": 0.3756929934024811, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5242674350738525, "sampling/sampling_logp_difference/mean": 0.013873964548110962, "step": 1495, "step_time": 75.7975627630949 }, { "clip_ratio/high_max": 0.01448886247817427, "clip_ratio/high_mean": 0.0040188360726460814, "clip_ratio/low_mean": 0.0016113281089928932, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005630164087051526, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 943.0, "completions/mean_length": 795.2250366210938, "completions/mean_terminated_length": 542.368408203125, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.12160459253937006, "epoch": 0.850967007963595, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04271277040243149, "kl": 0.014660390675999224, "learning_rate": 1.3464163822525597e-06, "loss": -0.0474, "num_tokens": 157842824.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.827622175216675, "sampling/importance_sampling_ratio/mean": 0.475600004196167, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3840200901031494, "sampling/sampling_logp_difference/mean": 0.015535685233771801, "step": 1496, "step_time": 74.09020292107016 }, { "clip_ratio/high_max": 0.014150727016385645, "clip_ratio/high_mean": 0.005164680114830844, "clip_ratio/low_mean": 0.002222957569756545, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00738763774279505, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 807.4750366210938, "completions/mean_terminated_length": 630.3181762695312, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.13339795917272568, "epoch": 0.8515358361774744, "frac_reward_zero_std": 0.0, "grad_norm": 0.019692782312631607, "kl": 0.014311795239336789, "learning_rate": 1.3412969283276449e-06, "loss": -0.0233, "num_tokens": 157942459.0, "reward": 1.024999976158142, "reward_std": 0.9736924767494202, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.436281204223633, "sampling/importance_sampling_ratio/mean": 0.4109874367713928, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9139535427093506, "sampling/sampling_logp_difference/mean": 0.014737670309841633, "step": 1497, "step_time": 72.52274065488018 }, { "clip_ratio/high_max": 0.005620051582809538, "clip_ratio/high_mean": 0.0011240103049203753, "clip_ratio/low_mean": 0.002441406228172127, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003565416467608884, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 699.0750122070312, "completions/mean_terminated_length": 405.0952453613281, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.11749558988958597, "epoch": 0.8521046643913538, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024628665298223495, "kl": 0.01140975160524249, "learning_rate": 1.3361774744027303e-06, "loss": 0.0174, "num_tokens": 158045894.0, "reward": 0.8500000238418579, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.6142871379852295, "sampling/importance_sampling_ratio/mean": 0.5236622095108032, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.031245470046997, "sampling/sampling_logp_difference/mean": 0.01460463460534811, "step": 1498, "step_time": 75.1666889928747 }, { "clip_ratio/high_max": 0.013338091084733605, "clip_ratio/high_mean": 0.0040579856431577355, "clip_ratio/low_mean": 0.00031738281541038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0043753684585681185, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 788.0, "completions/mean_length": 669.4750366210938, "completions/mean_terminated_length": 535.0, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.10033474769443274, "epoch": 0.8526734926052332, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009503493085503578, "kl": 0.013174868188798428, "learning_rate": 1.3310580204778155e-06, "loss": 0.016, "num_tokens": 158138905.0, "reward": 1.4500000476837158, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.7250000238418579, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.69207763671875, "sampling/importance_sampling_ratio/mean": 0.3782669007778168, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1508285999298096, "sampling/sampling_logp_difference/mean": 0.014046422205865383, "step": 1499, "step_time": 74.15458690421656 }, { "clip_ratio/high_max": 0.00898306374438107, "clip_ratio/high_mean": 0.002151642314856872, "clip_ratio/low_mean": 0.0008544922020519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030061344878049567, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 961.0, "completions/mean_length": 771.2000122070312, "completions/mean_terminated_length": 584.3478393554688, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.08187747513875365, "epoch": 0.8532423208191127, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.006771387066692114, "kl": 0.01285433059092611, "learning_rate": 1.325938566552901e-06, "loss": -0.0052, "num_tokens": 158242569.0, "reward": 1.024999976158142, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.606688141822815, "sampling/importance_sampling_ratio/mean": 0.2326958179473877, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7609100341796875, "sampling/sampling_logp_difference/mean": 0.012604504823684692, "step": 1500, "step_time": 73.7636700249277 }, { "clip_ratio/high_max": 0.015565202629659325, "clip_ratio/high_mean": 0.0054968992044450715, "clip_ratio/low_mean": 0.000854492205689894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006351391464704648, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 929.0, "completions/mean_length": 813.2750244140625, "completions/mean_terminated_length": 622.6190795898438, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.0818834314122796, "epoch": 0.853811149032992, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02961752749979496, "kl": 0.01172367762774229, "learning_rate": 1.3208191126279862e-06, "loss": 0.0008, "num_tokens": 158353380.0, "reward": 1.0500000715255737, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057363510131836, "sampling/importance_sampling_ratio/max": 2.6291391849517822, "sampling/importance_sampling_ratio/mean": 0.45043477416038513, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5556700229644775, "sampling/sampling_logp_difference/mean": 0.0118241673335433, "step": 1501, "step_time": 76.27350800740533 }, { "clip_ratio/high_max": 0.01409194606821984, "clip_ratio/high_mean": 0.0040431285597151145, "clip_ratio/low_mean": 0.002026367190410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006069495866540819, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 874.8500366210938, "completions/mean_terminated_length": 481.6363830566406, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.11800524685531855, "epoch": 0.8543799772468714, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.025588620454072952, "kl": 0.010688581911381334, "learning_rate": 1.3156996587030716e-06, "loss": 0.1114, "num_tokens": 158464910.0, "reward": 0.4000000059604645, "reward_std": 0.7089175581932068, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.7464945316314697, "sampling/importance_sampling_ratio/mean": 0.4487386643886566, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8146400451660156, "sampling/sampling_logp_difference/mean": 0.012801374308764935, "step": 1502, "step_time": 76.97501743165776 }, { "clip_ratio/high_max": 0.013903184095397592, "clip_ratio/high_mean": 0.005670002748956904, "clip_ratio/low_mean": 0.0027343750443833414, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008404377673286945, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 868.7750244140625, "completions/mean_terminated_length": 741.7727661132812, "completions/min_length": 475.0, "completions/min_terminated_length": 475.0, "entropy": 0.14695980120450258, "epoch": 0.8549488054607508, "frac_reward_zero_std": 0.0, "grad_norm": 0.03630172461271286, "kl": 0.01381738786585629, "learning_rate": 1.3105802047781569e-06, "loss": 0.0265, "num_tokens": 158567125.0, "reward": 1.024999976158142, "reward_std": 0.9736924767494202, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.566741704940796, "sampling/importance_sampling_ratio/mean": 0.487809956073761, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.178978443145752, "sampling/sampling_logp_difference/mean": 0.01648794859647751, "step": 1503, "step_time": 72.69768744893372 }, { "clip_ratio/high_max": 0.013650756445713341, "clip_ratio/high_mean": 0.005388906312873587, "clip_ratio/low_mean": 0.002254343213280663, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007643249584361911, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 920.0, "completions/mean_length": 712.0250244140625, "completions/mean_terminated_length": 504.04168701171875, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.12501113209873438, "epoch": 0.8555176336746303, "frac_reward_zero_std": 0.0, "grad_norm": 0.09941822290420532, "kl": 0.018689588643610477, "learning_rate": 1.3054607508532423e-06, "loss": -0.1495, "num_tokens": 158670854.0, "reward": 1.0750000476837158, "reward_std": 0.9442809820175171, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.780236005783081, "sampling/importance_sampling_ratio/mean": 0.633437991142273, "sampling/importance_sampling_ratio/min": 0.0010464079678058624, "sampling/sampling_logp_difference/max": 3.0114071369171143, "sampling/sampling_logp_difference/mean": 0.014677613973617554, "step": 1504, "step_time": 76.09470186335966 }, { "clip_ratio/high_max": 0.011971076019108295, "clip_ratio/high_mean": 0.0039864949794719, "clip_ratio/low_mean": 0.001196289074869128, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005182784079806879, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 875.4500122070312, "completions/mean_terminated_length": 674.4705810546875, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "entropy": 0.07065752986818552, "epoch": 0.8560864618885097, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013694856315851212, "kl": 0.009169288212433457, "learning_rate": 1.3003412969283277e-06, "loss": 0.0709, "num_tokens": 158780112.0, "reward": 0.800000011920929, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.958538293838501, "sampling/importance_sampling_ratio/mean": 0.328416109085083, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.103261947631836, "sampling/sampling_logp_difference/mean": 0.00988172460347414, "step": 1505, "step_time": 75.33083404018544 }, { "clip_ratio/high_max": 0.008791013387963176, "clip_ratio/high_mean": 0.0033240980410482734, "clip_ratio/low_mean": 0.00228503728430951, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005609135259874165, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 816.25, "completions/mean_terminated_length": 691.5999755859375, "completions/min_length": 419.0, "completions/min_terminated_length": 419.0, "entropy": 0.12874336820095778, "epoch": 0.856655290102389, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.034065090119838715, "kl": 0.014073442900553346, "learning_rate": 1.295221843003413e-06, "loss": -0.0813, "num_tokens": 158891106.0, "reward": 1.1750000715255737, "reward_std": 0.9577617645263672, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.9054875373840332, "sampling/importance_sampling_ratio/mean": 0.39686357975006104, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3506851196289062, "sampling/sampling_logp_difference/mean": 0.016345718875527382, "step": 1506, "step_time": 75.27763513894752 }, { "clip_ratio/high_max": 0.008264571893960238, "clip_ratio/high_mean": 0.002502115356037393, "clip_ratio/low_mean": 0.00127096964570228, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037730850162915885, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 1024.0, "completions/max_terminated_length": 897.0, "completions/mean_length": 587.375, "completions/mean_terminated_length": 510.3235168457031, "completions/min_length": 190.0, "completions/min_terminated_length": 190.0, "entropy": 0.12542972713708878, "epoch": 0.8572241183162684, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.015495614148676395, "kl": 0.014127425733022392, "learning_rate": 1.2901023890784984e-06, "loss": 0.049, "num_tokens": 158974609.0, "reward": 1.4500000476837158, "reward_std": 0.7493587136268616, "rewards/format_reward/mean": 0.8500000238418579, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.6308155059814453, "sampling/importance_sampling_ratio/mean": 0.342591255903244, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.445136308670044, "sampling/sampling_logp_difference/mean": 0.017200574278831482, "step": 1507, "step_time": 67.06461677909829 }, { "clip_ratio/high_max": 0.013234767131507397, "clip_ratio/high_mean": 0.003518038385664113, "clip_ratio/low_mean": 0.002059930357063422, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005577968840952963, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 774.9249877929688, "completions/mean_terminated_length": 608.875, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.1054671504534781, "epoch": 0.8577929465301479, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027312716469168663, "kl": 0.01069441850995645, "learning_rate": 1.2849829351535836e-06, "loss": 0.0139, "num_tokens": 159059830.0, "reward": 0.949999988079071, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.1330580711364746, "sampling/importance_sampling_ratio/mean": 0.5438222885131836, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0900299549102783, "sampling/sampling_logp_difference/mean": 0.01235479861497879, "step": 1508, "step_time": 68.20242627640255 }, { "clip_ratio/high_max": 0.014060610206797719, "clip_ratio/high_mean": 0.004089617475983687, "clip_ratio/low_mean": 0.0024414062518189894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006531023653224111, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 947.0, "completions/mean_length": 892.5499877929688, "completions/mean_terminated_length": 619.5385131835938, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.16377479303628206, "epoch": 0.8583617747440273, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.012702999636530876, "kl": 0.014383779605850577, "learning_rate": 1.279863481228669e-06, "loss": -0.0224, "num_tokens": 159170380.0, "reward": 0.6000000238418579, "reward_std": 0.9001424312591553, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 1.9565768241882324, "sampling/importance_sampling_ratio/mean": 0.21207423508167267, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7350397109985352, "sampling/sampling_logp_difference/mean": 0.016852138563990593, "step": 1509, "step_time": 75.5070133979898 }, { "clip_ratio/high_max": 0.009608407272025943, "clip_ratio/high_mean": 0.002677539123396855, "clip_ratio/low_mean": 0.002001378539716825, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004678917583078146, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 912.0, "completions/mean_length": 620.6749877929688, "completions/mean_terminated_length": 519.84375, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.11363812070339918, "epoch": 0.8589306029579067, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.028571339324116707, "kl": 0.014521983452141285, "learning_rate": 1.2747440273037543e-06, "loss": 0.0019, "num_tokens": 159265079.0, "reward": 1.4500000476837158, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.9881738424301147, "sampling/importance_sampling_ratio/mean": 0.46626386046409607, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.096489667892456, "sampling/sampling_logp_difference/mean": 0.014800828881561756, "step": 1510, "step_time": 72.14238685299642 }, { "clip_ratio/high_max": 0.013148569443728775, "clip_ratio/high_mean": 0.004361423343652859, "clip_ratio/low_mean": 0.002566093775385525, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00692751700989902, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 821.7750244140625, "completions/mean_terminated_length": 574.6111450195312, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.07442908687517047, "epoch": 0.8594994311717861, "frac_reward_zero_std": 0.0, "grad_norm": 0.019566386938095093, "kl": 0.012878784269560128, "learning_rate": 1.2696245733788397e-06, "loss": 0.0678, "num_tokens": 159364910.0, "reward": 0.75, "reward_std": 0.8987169861793518, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.9287757873535156, "sampling/importance_sampling_ratio/mean": 0.3315797448158264, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8096017837524414, "sampling/sampling_logp_difference/mean": 0.010327055118978024, "step": 1511, "step_time": 71.05448852665722 }, { "clip_ratio/high_max": 0.009804410801734775, "clip_ratio/high_mean": 0.0021400936238933355, "clip_ratio/low_mean": 0.0009033203132275958, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030434139480348676, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 847.0, "completions/mean_length": 713.3500366210938, "completions/mean_terminated_length": 506.25, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.07286469684913754, "epoch": 0.8600682593856656, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009070300497114658, "kl": 0.013148379744961858, "learning_rate": 1.264505119453925e-06, "loss": 0.0339, "num_tokens": 159465292.0, "reward": 1.2000000476837158, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.5789129734039307, "sampling/importance_sampling_ratio/mean": 0.4802541136741638, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.555668354034424, "sampling/sampling_logp_difference/mean": 0.01160286832600832, "step": 1512, "step_time": 73.368619737681 }, { "clip_ratio/high_max": 0.015555774560198188, "clip_ratio/high_mean": 0.006712899135891348, "clip_ratio/low_mean": 0.003018125134985894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009731024387292564, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 958.0, "completions/mean_length": 805.7000122070312, "completions/mean_terminated_length": 660.1666870117188, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.13309935852885246, "epoch": 0.8606370875995449, "frac_reward_zero_std": 0.0, "grad_norm": 0.03907962143421173, "kl": 0.030169761856086552, "learning_rate": 1.2593856655290104e-06, "loss": -0.0678, "num_tokens": 159573568.0, "reward": 1.125, "reward_std": 0.9919548630714417, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.4826369285583496, "sampling/importance_sampling_ratio/mean": 0.4672256112098694, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.179722309112549, "sampling/sampling_logp_difference/mean": 0.016507938504219055, "step": 1513, "step_time": 77.69023064686917 }, { "clip_ratio/high_max": 0.008215294394176453, "clip_ratio/high_mean": 0.001830184628488496, "clip_ratio/low_mean": 0.0019287109753349796, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037588955892715603, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 884.7000122070312, "completions/mean_terminated_length": 652.5333862304688, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.11091091856360435, "epoch": 0.8612059158134243, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.036077067255973816, "kl": 0.013246537651866674, "learning_rate": 1.2542662116040956e-06, "loss": -0.0719, "num_tokens": 159684180.0, "reward": 0.5250000357627869, "reward_std": 0.7506407499313354, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.894735813140869, "sampling/importance_sampling_ratio/mean": 0.4200642704963684, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.326719284057617, "sampling/sampling_logp_difference/mean": 0.014234972186386585, "step": 1514, "step_time": 74.61732473038137 }, { "clip_ratio/high_max": 0.012503402074798942, "clip_ratio/high_mean": 0.0033723370142979547, "clip_ratio/low_mean": 0.0026123046700377017, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005984641640679911, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 916.0, "completions/mean_length": 918.75, "completions/mean_terminated_length": 603.0, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.09539662022143602, "epoch": 0.8617747440273038, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015567642636597157, "kl": 0.009279502381104976, "learning_rate": 1.249146757679181e-06, "loss": 0.0253, "num_tokens": 159792682.0, "reward": 0.5, "reward_std": 0.8770580291748047, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.2342288494110107, "sampling/importance_sampling_ratio/mean": 0.2893069386482239, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.1765360832214355, "sampling/sampling_logp_difference/mean": 0.012402500957250595, "step": 1515, "step_time": 77.15644850488752 }, { "clip_ratio/high_max": 0.012436549994163215, "clip_ratio/high_mean": 0.0045137676352169365, "clip_ratio/low_mean": 0.0034891918767243624, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.008002959482837468, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 845.0250244140625, "completions/mean_terminated_length": 712.7391357421875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.1792888157069683, "epoch": 0.8623435722411832, "frac_reward_zero_std": 0.0, "grad_norm": 0.025301950052380562, "kl": 0.014054260798729956, "learning_rate": 1.2440273037542662e-06, "loss": -0.0542, "num_tokens": 159901323.0, "reward": 0.925000011920929, "reward_std": 0.8883144855499268, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.3522794246673584, "sampling/importance_sampling_ratio/mean": 0.37019768357276917, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9315236806869507, "sampling/sampling_logp_difference/mean": 0.01811073161661625, "step": 1516, "step_time": 78.06119206501171 }, { "clip_ratio/high_max": 0.011230702861212194, "clip_ratio/high_mean": 0.0032774193095974624, "clip_ratio/low_mean": 0.0017822265799622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00505964586045593, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 812.6749877929688, "completions/mean_terminated_length": 621.4761962890625, "completions/min_length": 393.0, "completions/min_terminated_length": 393.0, "entropy": 0.095547866076231, "epoch": 0.8629124004550626, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.038674816489219666, "kl": 0.01147837657481432, "learning_rate": 1.2389078498293517e-06, "loss": -0.0604, "num_tokens": 160006534.0, "reward": 0.699999988079071, "reward_std": 0.7578647136688232, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.7325732707977295, "sampling/importance_sampling_ratio/mean": 0.38362035155296326, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5283570289611816, "sampling/sampling_logp_difference/mean": 0.012491153553128242, "step": 1517, "step_time": 72.20511206123047 }, { "clip_ratio/high_max": 0.012797505245544016, "clip_ratio/high_mean": 0.0052121642511337996, "clip_ratio/low_mean": 0.001977565963898087, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.007189730182290077, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 963.0, "completions/mean_length": 831.0, "completions/mean_terminated_length": 673.0909423828125, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "entropy": 0.08113783039152622, "epoch": 0.863481228668942, "frac_reward_zero_std": 0.0, "grad_norm": 0.028465908020734787, "kl": 0.012431360664777458, "learning_rate": 1.2337883959044369e-06, "loss": -0.0807, "num_tokens": 160109958.0, "reward": 1.024999976158142, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.6419429779052734, "sampling/importance_sampling_ratio/mean": 0.38444212079048157, "sampling/importance_sampling_ratio/min": 0.00435743760317564, "sampling/sampling_logp_difference/max": 3.05566668510437, "sampling/sampling_logp_difference/mean": 0.011712263338267803, "step": 1518, "step_time": 71.58522083517164 }, { "clip_ratio/high_max": 0.009399957023561, "clip_ratio/high_mean": 0.002333679440198466, "clip_ratio/low_mean": 0.003735351550858468, "clip_ratio/low_min": 0.000732421875, "clip_ratio/region_mean": 0.006069030932849273, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 973.5499877929688, "completions/mean_terminated_length": 735.7142944335938, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.10345166549086571, "epoch": 0.8640500568828214, "frac_reward_zero_std": 0.0, "grad_norm": 0.014040837064385414, "kl": 0.009148526645731181, "learning_rate": 1.2286689419795223e-06, "loss": 0.0314, "num_tokens": 160221372.0, "reward": 0.30000001192092896, "reward_std": 0.6868732571601868, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.261561155319214, "sampling/importance_sampling_ratio/mean": 0.27892905473709106, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9373831748962402, "sampling/sampling_logp_difference/mean": 0.011946158483624458, "step": 1519, "step_time": 74.40657601878047 }, { "clip_ratio/high_max": 0.0052467541536316276, "clip_ratio/high_mean": 0.0012638116095331497, "clip_ratio/low_mean": 0.0019531250145519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032169366313610226, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 857.0, "completions/mean_length": 930.2250366210938, "completions/mean_terminated_length": 735.4615478515625, "completions/min_length": 555.0, "completions/min_terminated_length": 555.0, "entropy": 0.09827819932252169, "epoch": 0.8646188850967008, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.035363368690013885, "kl": 0.009513696888461709, "learning_rate": 1.2235494880546075e-06, "loss": 0.0724, "num_tokens": 160314621.0, "reward": 0.5, "reward_std": 0.7844645380973816, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.411817789077759, "sampling/importance_sampling_ratio/mean": 0.5767771005630493, "sampling/importance_sampling_ratio/min": 0.008245060220360756, "sampling/sampling_logp_difference/max": 2.009868621826172, "sampling/sampling_logp_difference/mean": 0.01191073190420866, "step": 1520, "step_time": 72.42238464672118 }, { "clip_ratio/high_max": 0.013414087006822228, "clip_ratio/high_mean": 0.003202142077498138, "clip_ratio/low_mean": 0.001489257840148639, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0046913999249227345, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 953.0, "completions/mean_length": 863.7999877929688, "completions/mean_terminated_length": 623.5, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.09560586139559746, "epoch": 0.8651877133105802, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015278005972504616, "kl": 0.009116710454691201, "learning_rate": 1.218430034129693e-06, "loss": 0.0533, "num_tokens": 160424925.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.2199432849884033, "sampling/importance_sampling_ratio/mean": 0.27848806977272034, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1608407497406006, "sampling/sampling_logp_difference/mean": 0.011564270593225956, "step": 1521, "step_time": 76.49400256108493 }, { "clip_ratio/high_max": 0.011395873269066215, "clip_ratio/high_mean": 0.003239856014261022, "clip_ratio/low_mean": 0.0029052733589196578, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006145129300421104, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 876.0, "completions/mean_length": 941.375, "completions/mean_terminated_length": 693.5, "completions/min_length": 469.0, "completions/min_terminated_length": 469.0, "entropy": 0.1207658825442195, "epoch": 0.8657565415244596, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03318561986088753, "kl": 0.011476582614704967, "learning_rate": 1.2133105802047782e-06, "loss": -0.1102, "num_tokens": 160532732.0, "reward": 0.42500001192092896, "reward_std": 0.780778706073761, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.784592390060425, "sampling/importance_sampling_ratio/mean": 0.358806312084198, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.104288101196289, "sampling/sampling_logp_difference/mean": 0.013816380873322487, "step": 1522, "step_time": 77.60679998947307 }, { "clip_ratio/high_max": 0.01043810840928927, "clip_ratio/high_mean": 0.002301296881341841, "clip_ratio/low_mean": 0.00046386719441215973, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0027651639975374565, "completions/clipped_ratio": 0.17499999701976776, "completions/max_length": 1024.0, "completions/max_terminated_length": 908.0, "completions/mean_length": 622.0750122070312, "completions/mean_terminated_length": 536.8181762695312, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.08947554789483547, "epoch": 0.8663253697383391, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.018219882622361183, "kl": 0.018303390475921333, "learning_rate": 1.2081911262798636e-06, "loss": -0.0361, "num_tokens": 160630359.0, "reward": 1.649999976158142, "reward_std": 0.7696152925491333, "rewards/format_reward/mean": 0.824999988079071, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.824999988079071, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.9382388591766357, "sampling/importance_sampling_ratio/mean": 0.4256901144981384, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.4375221729278564, "sampling/sampling_logp_difference/mean": 0.013596782460808754, "step": 1523, "step_time": 72.17817793902941 }, { "clip_ratio/high_max": 0.010664172994438559, "clip_ratio/high_mean": 0.00301074280650937, "clip_ratio/low_mean": 0.0011962890857830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004207031859550625, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 776.2750244140625, "completions/mean_terminated_length": 573.5909423828125, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.08356596669182181, "epoch": 0.8668941979522184, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025566745549440384, "kl": 0.01229967107065022, "learning_rate": 1.2030716723549489e-06, "loss": -0.1064, "num_tokens": 160734810.0, "reward": 1.100000023841858, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.8953585624694824, "sampling/importance_sampling_ratio/mean": 0.5754833221435547, "sampling/importance_sampling_ratio/min": 0.00494904862716794, "sampling/sampling_logp_difference/max": 2.0972752571105957, "sampling/sampling_logp_difference/mean": 0.01225754152983427, "step": 1524, "step_time": 73.1499115228653 }, { "clip_ratio/high_max": 0.013513340323697776, "clip_ratio/high_mean": 0.004001252083980944, "clip_ratio/low_mean": 0.001782226565410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0057834786712192, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 799.0, "completions/mean_length": 758.0750122070312, "completions/mean_terminated_length": 492.1499938964844, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.09650398511439562, "epoch": 0.8674630261660978, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.021950023248791695, "kl": 0.01342967664822936, "learning_rate": 1.1979522184300343e-06, "loss": 0.0189, "num_tokens": 160841749.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.5816290378570557, "sampling/importance_sampling_ratio/mean": 0.5045166015625, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9632748365402222, "sampling/sampling_logp_difference/mean": 0.014000199735164642, "step": 1525, "step_time": 73.56247359141707 }, { "clip_ratio/high_max": 0.012831572908908129, "clip_ratio/high_mean": 0.003630848994362168, "clip_ratio/low_mean": 0.0015319421945605427, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005162791116163135, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 878.5750122070312, "completions/mean_terminated_length": 681.8235473632812, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.12228028476238251, "epoch": 0.8680318543799772, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04627993330359459, "kl": 0.012103787972591817, "learning_rate": 1.1928327645051195e-06, "loss": -0.0202, "num_tokens": 160949884.0, "reward": 0.7750000357627869, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.766735315322876, "sampling/importance_sampling_ratio/mean": 0.4238017499446869, "sampling/importance_sampling_ratio/min": 0.0016814236296340823, "sampling/sampling_logp_difference/max": 3.6312761306762695, "sampling/sampling_logp_difference/mean": 0.013918782584369183, "step": 1526, "step_time": 74.69800430396572 }, { "clip_ratio/high_max": 0.011204141221242025, "clip_ratio/high_mean": 0.0033127510541817173, "clip_ratio/low_mean": 0.0029052734389551915, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.006218024529516697, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 929.1000366210938, "completions/mean_terminated_length": 707.6666870117188, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.1167455967515707, "epoch": 0.8686006825938567, "frac_reward_zero_std": 0.0, "grad_norm": 0.030633607879281044, "kl": 0.010039574990514666, "learning_rate": 1.187713310580205e-06, "loss": -0.1357, "num_tokens": 161060192.0, "reward": 0.45000001788139343, "reward_std": 0.7493587136268616, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.116382122039795, "sampling/importance_sampling_ratio/mean": 0.28506413102149963, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.355438470840454, "sampling/sampling_logp_difference/mean": 0.012708882801234722, "step": 1527, "step_time": 78.83320660772733 }, { "clip_ratio/high_max": 0.006963860243558884, "clip_ratio/high_mean": 0.001749205111991614, "clip_ratio/low_mean": 0.0018798828423314262, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003629087994340807, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 914.0, "completions/mean_length": 882.2000122070312, "completions/mean_terminated_length": 669.5, "completions/min_length": 418.0, "completions/min_terminated_length": 418.0, "entropy": 0.11460271570831537, "epoch": 0.8691695108077361, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014832284301519394, "kl": 0.01165128342108801, "learning_rate": 1.1825938566552902e-06, "loss": 0.0034, "num_tokens": 161162936.0, "reward": 0.75, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.8868796825408936, "sampling/importance_sampling_ratio/mean": 0.3580038249492645, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2225773334503174, "sampling/sampling_logp_difference/mean": 0.013385857455432415, "step": 1528, "step_time": 75.45865363813937 }, { "clip_ratio/high_max": 0.012842769850976765, "clip_ratio/high_mean": 0.004064907741849311, "clip_ratio/low_mean": 0.002377902543230448, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006442810292355716, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 859.625, "completions/mean_terminated_length": 613.0625, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.13160412292927504, "epoch": 0.8697383390216155, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04616977646946907, "kl": 0.010403883876278996, "learning_rate": 1.1774744027303756e-06, "loss": 0.16, "num_tokens": 161264513.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.657439708709717, "sampling/importance_sampling_ratio/mean": 0.5711872577667236, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7740755081176758, "sampling/sampling_logp_difference/mean": 0.01437234878540039, "step": 1529, "step_time": 73.45244185719639 }, { "clip_ratio/high_max": 0.012060958979418501, "clip_ratio/high_mean": 0.0030004787840880454, "clip_ratio/low_mean": 0.0010253906293655746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004025869391625747, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 871.0, "completions/mean_length": 911.375, "completions/mean_terminated_length": 573.5, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 0.08756216429173946, "epoch": 0.8703071672354948, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.03874722495675087, "kl": 0.010392509517259896, "learning_rate": 1.1723549488054608e-06, "loss": 0.0665, "num_tokens": 161379800.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.990062952041626, "sampling/importance_sampling_ratio/mean": 0.6040882468223572, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.670398235321045, "sampling/sampling_logp_difference/mean": 0.011889173649251461, "step": 1530, "step_time": 77.39431683835573 }, { "clip_ratio/high_max": 0.013201794354245067, "clip_ratio/high_mean": 0.004534718027571216, "clip_ratio/low_mean": 0.0010564237309154123, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0055911418166942894, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 960.0, "completions/mean_length": 849.9249877929688, "completions/mean_terminated_length": 614.4117431640625, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.13363250344991684, "epoch": 0.8708759954493743, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01789247803390026, "kl": 0.013839926570653915, "learning_rate": 1.167235494880546e-06, "loss": -0.0701, "num_tokens": 161484221.0, "reward": 0.824999988079071, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.0578787326812744, "sampling/importance_sampling_ratio/mean": 0.23458433151245117, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.455265760421753, "sampling/sampling_logp_difference/mean": 0.015152374282479286, "step": 1531, "step_time": 73.49273329181597 }, { "clip_ratio/high_max": 0.00896498968359083, "clip_ratio/high_mean": 0.0019795740954577923, "clip_ratio/low_mean": 0.000927734377910383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00290730845881626, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 890.0, "completions/mean_length": 752.6000366210938, "completions/mean_terminated_length": 589.760009765625, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.08424873370677233, "epoch": 0.8714448236632537, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.02042926661670208, "kl": 0.010647089802660048, "learning_rate": 1.1621160409556313e-06, "loss": 0.0204, "num_tokens": 161589509.0, "reward": 1.25, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.636280059814453, "sampling/importance_sampling_ratio/mean": 0.5484203100204468, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9799308776855469, "sampling/sampling_logp_difference/mean": 0.012087695300579071, "step": 1532, "step_time": 72.60033134394325 }, { "clip_ratio/high_max": 0.010117298021214083, "clip_ratio/high_mean": 0.0022594318870687857, "clip_ratio/low_mean": 0.0007568359451397555, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003016267830389552, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 967.0, "completions/mean_length": 724.1749877929688, "completions/mean_terminated_length": 544.2799682617188, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.14086647890508175, "epoch": 0.8720136518771331, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.00975471455603838, "kl": 0.020144084352068603, "learning_rate": 1.1569965870307167e-06, "loss": -0.0068, "num_tokens": 161693404.0, "reward": 1.25, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.44061017036438, "sampling/importance_sampling_ratio/mean": 0.36741703748703003, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.561520576477051, "sampling/sampling_logp_difference/mean": 0.01788952387869358, "step": 1533, "step_time": 72.14309790870175 }, { "clip_ratio/high_max": 0.012741054175421596, "clip_ratio/high_mean": 0.0033179147576447576, "clip_ratio/low_mean": 0.001391601586874458, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004709516215370968, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 872.0, "completions/mean_length": 912.1500244140625, "completions/mean_terminated_length": 576.6000366210938, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.07047422416508198, "epoch": 0.8725824800910125, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0086801303550601, "kl": 0.012886543816421181, "learning_rate": 1.151877133105802e-06, "loss": 0.0034, "num_tokens": 161805490.0, "reward": 0.5, "reward_std": 0.8770580887794495, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.3198370933532715, "sampling/importance_sampling_ratio/mean": 0.26208335161209106, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.053820610046387, "sampling/sampling_logp_difference/mean": 0.010750365443527699, "step": 1534, "step_time": 76.47160186618567 }, { "clip_ratio/high_max": 0.012560778413899243, "clip_ratio/high_mean": 0.0037080949259689078, "clip_ratio/low_mean": 0.0006347656344587449, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004342860571341589, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 974.0, "completions/mean_length": 819.5, "completions/mean_terminated_length": 615.0, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.10366429388523102, "epoch": 0.873151308304892, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.02285902202129364, "kl": 0.015246425871737301, "learning_rate": 1.1467576791808874e-06, "loss": 0.0453, "num_tokens": 161901630.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.4018142223358154, "sampling/importance_sampling_ratio/mean": 0.46169358491897583, "sampling/importance_sampling_ratio/min": 0.002075840486213565, "sampling/sampling_logp_difference/max": 2.318960428237915, "sampling/sampling_logp_difference/mean": 0.012775554321706295, "step": 1535, "step_time": 76.55146988900378 }, { "clip_ratio/high_max": 0.017365204286761582, "clip_ratio/high_mean": 0.006905103131430224, "clip_ratio/low_mean": 0.0023621291329618543, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00926723243901506, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 810.2000122070312, "completions/mean_terminated_length": 616.7619018554688, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.16773789003491402, "epoch": 0.8737201365187713, "frac_reward_zero_std": 0.0, "grad_norm": 0.020562948659062386, "kl": 0.014444725995417684, "learning_rate": 1.1416382252559726e-06, "loss": 0.0283, "num_tokens": 162005830.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.558218479156494, "sampling/importance_sampling_ratio/mean": 0.3225986063480377, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.223402738571167, "sampling/sampling_logp_difference/mean": 0.017397118732333183, "step": 1536, "step_time": 76.29539132211357 }, { "clip_ratio/high_max": 0.008942987711634487, "clip_ratio/high_mean": 0.0024325403646798804, "clip_ratio/low_mean": 0.002099102064676117, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0045316424511838704, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 895.3500366210938, "completions/mean_terminated_length": 702.375, "completions/min_length": 424.0, "completions/min_terminated_length": 424.0, "entropy": 0.13907010573893785, "epoch": 0.8742889647326507, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.010606840252876282, "kl": 0.01129908231087029, "learning_rate": 1.136518771331058e-06, "loss": -0.0054, "num_tokens": 162114860.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.717102527618408, "sampling/importance_sampling_ratio/mean": 0.38219621777534485, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.27001953125, "sampling/sampling_logp_difference/mean": 0.015486503019928932, "step": 1537, "step_time": 75.33099188678898 }, { "clip_ratio/high_max": 0.011586158419959247, "clip_ratio/high_mean": 0.0025526164754410274, "clip_ratio/low_mean": 0.002636718789290171, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005189335235627368, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 920.0, "completions/mean_length": 888.1500244140625, "completions/mean_terminated_length": 480.6000061035156, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.09234134899452329, "epoch": 0.8748577929465301, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03175641968846321, "kl": 0.03283510287292302, "learning_rate": 1.1313993174061432e-06, "loss": -0.069, "num_tokens": 162211242.0, "reward": 0.4749999940395355, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.4446604251861572, "sampling/importance_sampling_ratio/mean": 0.4792144298553467, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.459984302520752, "sampling/sampling_logp_difference/mean": 0.012003750540316105, "step": 1538, "step_time": 70.21257373644039 }, { "clip_ratio/high_max": 0.015709211118519306, "clip_ratio/high_mean": 0.006235151580767706, "clip_ratio/low_mean": 0.0024902343648136593, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00872538611292839, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 795.1000366210938, "completions/mean_terminated_length": 588.0, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.13298188336193562, "epoch": 0.8754266211604096, "frac_reward_zero_std": 0.0, "grad_norm": 0.03427882120013237, "kl": 0.013408920960500836, "learning_rate": 1.1262798634812287e-06, "loss": 0.0233, "num_tokens": 162319310.0, "reward": 0.925000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.3131966590881348, "sampling/importance_sampling_ratio/mean": 0.5544456839561462, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2900500297546387, "sampling/sampling_logp_difference/mean": 0.015748029574751854, "step": 1539, "step_time": 74.52612721989863 }, { "clip_ratio/high_max": 0.00863169744843617, "clip_ratio/high_mean": 0.0022632704494753852, "clip_ratio/low_mean": 0.0009277343851863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0031910047837300226, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 988.0, "completions/mean_length": 625.4000244140625, "completions/mean_terminated_length": 525.75, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.12393452320247889, "epoch": 0.875995449374289, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.006040392443537712, "kl": 0.01448802393861115, "learning_rate": 1.1211604095563139e-06, "loss": 0.0113, "num_tokens": 162425262.0, "reward": 1.524999976158142, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.7250000238418579, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.6545040607452393, "sampling/importance_sampling_ratio/mean": 0.4436991214752197, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.09905743598938, "sampling/sampling_logp_difference/mean": 0.016562489792704582, "step": 1540, "step_time": 70.82538586319424 }, { "clip_ratio/high_max": 0.01251429069088772, "clip_ratio/high_mean": 0.005779591534519568, "clip_ratio/low_mean": 0.0015136718920985004, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007293263450264931, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 894.0, "completions/mean_length": 718.25, "completions/mean_terminated_length": 514.4166870117188, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.11167528294026852, "epoch": 0.8765642775881684, "frac_reward_zero_std": 0.0, "grad_norm": 0.024681510403752327, "kl": 0.016710373456589878, "learning_rate": 1.1160409556313993e-06, "loss": 0.029, "num_tokens": 162532568.0, "reward": 1.149999976158142, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.0984983444213867, "sampling/importance_sampling_ratio/mean": 0.35573387145996094, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.325003147125244, "sampling/sampling_logp_difference/mean": 0.013899775221943855, "step": 1541, "step_time": 76.90855157934129 }, { "clip_ratio/high_max": 0.01279620113200508, "clip_ratio/high_mean": 0.0053694966263719834, "clip_ratio/low_mean": 0.0034750269987853244, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008844523574225605, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 990.0, "completions/mean_length": 799.0750122070312, "completions/mean_terminated_length": 664.1199951171875, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.15207756124436855, "epoch": 0.8771331058020477, "frac_reward_zero_std": 0.0, "grad_norm": 0.03487740829586983, "kl": 0.014803203055635095, "learning_rate": 1.1109215017064845e-06, "loss": 0.0771, "num_tokens": 162627355.0, "reward": 0.9750000238418579, "reward_std": 0.8619447350502014, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.681673288345337, "sampling/importance_sampling_ratio/mean": 0.5394917726516724, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.498691439628601, "sampling/sampling_logp_difference/mean": 0.01740379072725773, "step": 1542, "step_time": 69.37504756334238 }, { "clip_ratio/high_max": 0.009527356713078916, "clip_ratio/high_mean": 0.0024494139361195266, "clip_ratio/low_mean": 0.0015869140697759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004036327998619527, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 944.0, "completions/mean_length": 837.2750244140625, "completions/mean_terminated_length": 557.1875, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.09312834404408932, "epoch": 0.8777019340159272, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.029583262279629707, "kl": 0.011146183591336012, "learning_rate": 1.10580204778157e-06, "loss": 0.0889, "num_tokens": 162738078.0, "reward": 0.7250000238418579, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.2471203804016113, "sampling/importance_sampling_ratio/mean": 0.38517865538597107, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.458036422729492, "sampling/sampling_logp_difference/mean": 0.012090248987078667, "step": 1543, "step_time": 76.09127548383549 }, { "clip_ratio/high_max": 0.00896041898522526, "clip_ratio/high_mean": 0.0027702920488081872, "clip_ratio/low_mean": 0.0009033203241415322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036736123729497194, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 1024.0, "completions/max_terminated_length": 977.0, "completions/mean_length": 602.4750366210938, "completions/mean_terminated_length": 528.0882568359375, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.1573921348899603, "epoch": 0.8782707622298066, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011260518804192543, "kl": 0.017136044683866203, "learning_rate": 1.1006825938566552e-06, "loss": 0.0173, "num_tokens": 162832417.0, "reward": 1.7000000476837158, "reward_std": 0.7232405543327332, "rewards/format_reward/mean": 0.8500000238418579, "rewards/format_reward/std": 0.36162030696868896, "rewards/gpt_judge_reward/mean": 0.8500000238418579, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.327666759490967, "sampling/importance_sampling_ratio/mean": 0.37126222252845764, "sampling/importance_sampling_ratio/min": 0.005106072872877121, "sampling/sampling_logp_difference/max": 3.477383613586426, "sampling/sampling_logp_difference/mean": 0.019905317574739456, "step": 1544, "step_time": 72.36481248564087 }, { "clip_ratio/high_max": 0.007866241765441373, "clip_ratio/high_mean": 0.0017662985119386576, "clip_ratio/low_mean": 0.0009521484444121597, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002718446994549595, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 690.2000122070312, "completions/mean_terminated_length": 467.66668701171875, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.07098541595041752, "epoch": 0.878839590443686, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.020282600075006485, "kl": 0.010546528967097402, "learning_rate": 1.0955631399317406e-06, "loss": 0.0385, "num_tokens": 162937449.0, "reward": 1.2000000476837158, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 1.78507399559021, "sampling/importance_sampling_ratio/mean": 0.4171878695487976, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0683979988098145, "sampling/sampling_logp_difference/mean": 0.011222928762435913, "step": 1545, "step_time": 74.72591195371933 }, { "clip_ratio/high_max": 0.008705176907824352, "clip_ratio/high_mean": 0.0021278940985212103, "clip_ratio/low_mean": 0.0017044535616150824, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038323477638186887, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 961.0, "completions/mean_length": 847.5, "completions/mean_terminated_length": 687.8095092773438, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.09742042142897844, "epoch": 0.8794084186575654, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.009147627279162407, "kl": 0.011245344416238368, "learning_rate": 1.0904436860068259e-06, "loss": 0.0111, "num_tokens": 163034493.0, "reward": 0.875, "reward_std": 0.9111137390136719, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.900665521621704, "sampling/importance_sampling_ratio/mean": 0.5090509653091431, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1797115802764893, "sampling/sampling_logp_difference/mean": 0.011843710206449032, "step": 1546, "step_time": 79.07698161690496 }, { "clip_ratio/high_max": 0.01414292969275266, "clip_ratio/high_mean": 0.005129835073603317, "clip_ratio/low_mean": 0.0011489237531350227, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006278758810367435, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 927.0, "completions/mean_length": 768.7000122070312, "completions/mean_terminated_length": 631.2307739257812, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.11035061627626419, "epoch": 0.8799772468714449, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.030740996822714806, "kl": 0.013446510245557874, "learning_rate": 1.0853242320819113e-06, "loss": -0.0752, "num_tokens": 163138457.0, "reward": 0.949999988079071, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.4429397583007812, "sampling/importance_sampling_ratio/mean": 0.36294931173324585, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.636634349822998, "sampling/sampling_logp_difference/mean": 0.014701115898787975, "step": 1547, "step_time": 73.58230661903508 }, { "clip_ratio/high_max": 0.006450381595641375, "clip_ratio/high_mean": 0.001723403125652112, "clip_ratio/low_mean": 0.0007568359560536919, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024802390980767086, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 825.0, "completions/mean_length": 855.75, "completions/mean_terminated_length": 575.3333740234375, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "entropy": 0.09334983583539724, "epoch": 0.8805460750853242, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011031831614673138, "kl": 0.010667117137927562, "learning_rate": 1.0802047781569965e-06, "loss": -0.0112, "num_tokens": 163247479.0, "reward": 0.699999988079071, "reward_std": 0.9391758441925049, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.4213948249816895, "sampling/importance_sampling_ratio/mean": 0.31930428743362427, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.854548692703247, "sampling/sampling_logp_difference/mean": 0.012266017496585846, "step": 1548, "step_time": 72.35493874666281 }, { "clip_ratio/high_max": 0.01271754945628345, "clip_ratio/high_mean": 0.004899355815723538, "clip_ratio/low_mean": 0.0011230468826397555, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006022402783855796, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 916.0, "completions/mean_length": 747.4750366210938, "completions/mean_terminated_length": 581.5599975585938, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.09137567970901728, "epoch": 0.8811149032992036, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03485424071550369, "kl": 0.016703722532838583, "learning_rate": 1.075085324232082e-06, "loss": 0.0806, "num_tokens": 163349666.0, "reward": 1.25, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.668642997741699, "sampling/importance_sampling_ratio/mean": 0.5437278151512146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.325228691101074, "sampling/sampling_logp_difference/mean": 0.012402358464896679, "step": 1549, "step_time": 70.46843408653513 }, { "clip_ratio/high_max": 0.013682582997716963, "clip_ratio/high_mean": 0.0035495975898811594, "clip_ratio/low_mean": 0.0015281232408597134, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0050777209689840674, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 918.75, "completions/mean_terminated_length": 700.1538696289062, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "entropy": 0.14986417442560196, "epoch": 0.8816837315130831, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.017048275098204613, "kl": 0.01285065698903054, "learning_rate": 1.0699658703071672e-06, "loss": 0.0637, "num_tokens": 163463616.0, "reward": 0.6000000238418579, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.42147159576416, "sampling/importance_sampling_ratio/mean": 0.21544115245342255, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.945829153060913, "sampling/sampling_logp_difference/mean": 0.016245147213339806, "step": 1550, "step_time": 77.53798218001612 }, { "clip_ratio/high_max": 0.011364997015334666, "clip_ratio/high_mean": 0.0042459975884412415, "clip_ratio/low_mean": 0.002221679678768851, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0064676772453822196, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 853.0499877929688, "completions/mean_terminated_length": 621.7647094726562, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.10025169234722853, "epoch": 0.8822525597269625, "frac_reward_zero_std": 0.0, "grad_norm": 0.023565838113427162, "kl": 0.013606222928501666, "learning_rate": 1.0648464163822526e-06, "loss": 0.0023, "num_tokens": 163564418.0, "reward": 0.7750000357627869, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.9573715925216675, "sampling/importance_sampling_ratio/mean": 0.29649752378463745, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.457134962081909, "sampling/sampling_logp_difference/mean": 0.012677199207246304, "step": 1551, "step_time": 71.02649022568949 }, { "clip_ratio/high_max": 0.014702585816849023, "clip_ratio/high_mean": 0.004421624253154732, "clip_ratio/low_mean": 0.0008640008400107035, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052856252586934716, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 931.0, "completions/mean_length": 732.9000244140625, "completions/mean_terminated_length": 517.7391357421875, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.08145348820835352, "epoch": 0.8828213879408419, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02312576398253441, "kl": 0.012068999116308987, "learning_rate": 1.059726962457338e-06, "loss": -0.0093, "num_tokens": 163665662.0, "reward": 1.100000023841858, "reward_std": 1.0076628923416138, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.328512668609619, "sampling/importance_sampling_ratio/mean": 0.3627813458442688, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.682567596435547, "sampling/sampling_logp_difference/mean": 0.012168033048510551, "step": 1552, "step_time": 70.9131741039455 }, { "clip_ratio/high_max": 0.015552999684587121, "clip_ratio/high_mean": 0.005349844286683947, "clip_ratio/low_mean": 0.0020339416369097307, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007383785909041762, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 961.0, "completions/mean_length": 873.25, "completions/mean_terminated_length": 689.0, "completions/min_length": 434.0, "completions/min_terminated_length": 434.0, "entropy": 0.1307593621313572, "epoch": 0.8833902161547212, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.028782540932297707, "kl": 0.011228370596654713, "learning_rate": 1.0546075085324233e-06, "loss": -0.0578, "num_tokens": 163773432.0, "reward": 0.8500000238418579, "reward_std": 0.9753369092941284, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.7256855964660645, "sampling/importance_sampling_ratio/mean": 0.5887510776519775, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.055063486099243, "sampling/sampling_logp_difference/mean": 0.015312336385250092, "step": 1553, "step_time": 75.39736384013668 }, { "clip_ratio/high_max": 0.015581762534566224, "clip_ratio/high_mean": 0.004945166816469282, "clip_ratio/low_mean": 0.0014404296853172127, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0063855964108370245, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 930.5, "completions/mean_terminated_length": 827.1578979492188, "completions/min_length": 511.0, "completions/min_terminated_length": 511.0, "entropy": 0.09423206932842731, "epoch": 0.8839590443686007, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0131914671510458, "kl": 0.012295496999286115, "learning_rate": 1.0494880546075087e-06, "loss": -0.0286, "num_tokens": 163880748.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.6194919347763062, "sampling/importance_sampling_ratio/mean": 0.24176093935966492, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4987072944641113, "sampling/sampling_logp_difference/mean": 0.012394688092172146, "step": 1554, "step_time": 74.25086561520584 }, { "clip_ratio/high_max": 0.015346842730650678, "clip_ratio/high_mean": 0.0072083590348483995, "clip_ratio/low_mean": 0.0022455877078755293, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009453946666326374, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 747.4750366210938, "completions/mean_terminated_length": 563.125, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.12051676958799362, "epoch": 0.8845278725824801, "frac_reward_zero_std": 0.0, "grad_norm": 0.02334846556186676, "kl": 0.015566077316179872, "learning_rate": 1.044368600682594e-06, "loss": 0.0001, "num_tokens": 163978831.0, "reward": 1.1750000715255737, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.8247101306915283, "sampling/importance_sampling_ratio/mean": 0.4206067621707916, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2118775844573975, "sampling/sampling_logp_difference/mean": 0.014394569210708141, "step": 1555, "step_time": 73.29845749936067 }, { "clip_ratio/high_max": 0.013443251722492278, "clip_ratio/high_mean": 0.005132181802764535, "clip_ratio/low_mean": 0.0016845703503349796, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006816752138547599, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 693.6500244140625, "completions/mean_terminated_length": 534.5925903320312, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.13481675274670124, "epoch": 0.8850967007963595, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015210599638521671, "kl": 0.02121634501963854, "learning_rate": 1.0392491467576794e-06, "loss": -0.0458, "num_tokens": 164084833.0, "reward": 1.274999976158142, "reward_std": 0.9333562850952148, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 1.8534687757492065, "sampling/importance_sampling_ratio/mean": 0.3340927064418793, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0307650566101074, "sampling/sampling_logp_difference/mean": 0.017537875100970268, "step": 1556, "step_time": 73.35566553100944 }, { "clip_ratio/high_max": 0.013957999995909631, "clip_ratio/high_mean": 0.005866617488209158, "clip_ratio/low_mean": 0.0011962890857830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0070629065739922225, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 944.0, "completions/mean_length": 770.7250366210938, "completions/mean_terminated_length": 517.4500122070312, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.09097287897020578, "epoch": 0.8856655290102389, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018865805119276047, "kl": 0.009606150386389345, "learning_rate": 1.0341296928327646e-06, "loss": 0.0147, "num_tokens": 164182630.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.479016661643982, "sampling/importance_sampling_ratio/mean": 0.375252366065979, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9835848808288574, "sampling/sampling_logp_difference/mean": 0.012120011262595654, "step": 1557, "step_time": 74.94116386305541 }, { "clip_ratio/high_max": 0.018965970259159803, "clip_ratio/high_mean": 0.005745881993789226, "clip_ratio/low_mean": 0.0017822265799622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007528108370024711, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 728.1500244140625, "completions/mean_terminated_length": 530.9166870117188, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.10704807750880718, "epoch": 0.8862343572241184, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01646098680794239, "kl": 0.011096767440903932, "learning_rate": 1.0290102389078498e-06, "loss": 0.0496, "num_tokens": 164279660.0, "reward": 1.2000000476837158, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.16609787940979, "sampling/importance_sampling_ratio/mean": 0.4427782893180847, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4354135990142822, "sampling/sampling_logp_difference/mean": 0.014274917542934418, "step": 1558, "step_time": 72.42740951152518 }, { "clip_ratio/high_max": 0.011999242939054966, "clip_ratio/high_mean": 0.004748609440866858, "clip_ratio/low_mean": 0.0023157177784014493, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00706432739389129, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 891.0, "completions/mean_length": 835.0750122070312, "completions/mean_terminated_length": 680.5, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.10282528959214687, "epoch": 0.8868031854379977, "frac_reward_zero_std": 0.0, "grad_norm": 0.022139467298984528, "kl": 0.011826276546344161, "learning_rate": 1.0238907849829352e-06, "loss": -0.0358, "num_tokens": 164381231.0, "reward": 1.024999976158142, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.5090444087982178, "sampling/importance_sampling_ratio/mean": 0.3342081606388092, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.194749116897583, "sampling/sampling_logp_difference/mean": 0.013346726074814796, "step": 1559, "step_time": 73.06120428303257 }, { "clip_ratio/high_max": 0.015590493567287922, "clip_ratio/high_mean": 0.004772836808115244, "clip_ratio/low_mean": 0.0033691405988065526, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.008141977421473712, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 917.0, "completions/mean_length": 874.2000122070312, "completions/mean_terminated_length": 624.5333862304688, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.1542069222778082, "epoch": 0.8873720136518771, "frac_reward_zero_std": 0.0, "grad_norm": 0.026173027232289314, "kl": 0.013769800192676485, "learning_rate": 1.0187713310580205e-06, "loss": 0.0072, "num_tokens": 164488087.0, "reward": 0.699999988079071, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.127004384994507, "sampling/importance_sampling_ratio/mean": 0.27682167291641235, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.568293571472168, "sampling/sampling_logp_difference/mean": 0.016539311036467552, "step": 1560, "step_time": 74.7759407311678 }, { "clip_ratio/high_max": 0.009360309923067689, "clip_ratio/high_mean": 0.002798994886688888, "clip_ratio/low_mean": 0.0016793326267361408, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004478327551623806, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 752.7250366210938, "completions/mean_terminated_length": 606.6538696289062, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.1388592440634966, "epoch": 0.8879408418657565, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03396333009004593, "kl": 0.013243775931186974, "learning_rate": 1.0136518771331059e-06, "loss": 0.1648, "num_tokens": 164589068.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.7342417240142822, "sampling/importance_sampling_ratio/mean": 0.5499018430709839, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3840184211730957, "sampling/sampling_logp_difference/mean": 0.01587221585214138, "step": 1561, "step_time": 71.99909337097779 }, { "clip_ratio/high_max": 0.007248631678521633, "clip_ratio/high_mean": 0.0019332845695316792, "clip_ratio/low_mean": 0.0010498046976863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029830892744939774, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 805.7250366210938, "completions/mean_terminated_length": 441.933349609375, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.08012676890939474, "epoch": 0.888509670079636, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.012692060321569443, "kl": 0.011233940429519862, "learning_rate": 1.0085324232081911e-06, "loss": -0.0352, "num_tokens": 164695337.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.4587464332580566, "sampling/importance_sampling_ratio/mean": 0.582804799079895, "sampling/importance_sampling_ratio/min": 0.006216893903911114, "sampling/sampling_logp_difference/max": 2.946713447570801, "sampling/sampling_logp_difference/mean": 0.01132928766310215, "step": 1562, "step_time": 71.2606515858788 }, { "clip_ratio/high_max": 0.007222051965072751, "clip_ratio/high_mean": 0.0024230122053268133, "clip_ratio/low_mean": 0.0008789062703726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003301918477518484, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 661.1749877929688, "completions/mean_terminated_length": 465.8077087402344, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 0.1385478051379323, "epoch": 0.8890784982935154, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.011469154618680477, "kl": 0.015441284631378949, "learning_rate": 1.0034129692832765e-06, "loss": -0.0154, "num_tokens": 164800328.0, "reward": 1.3000000715255737, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.162163019180298, "sampling/importance_sampling_ratio/mean": 0.5124218463897705, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6363685131073, "sampling/sampling_logp_difference/mean": 0.016372572630643845, "step": 1563, "step_time": 71.91557114804164 }, { "clip_ratio/high_max": 0.008283714763820171, "clip_ratio/high_mean": 0.0020707699877675623, "clip_ratio/low_mean": 0.0031982422369765118, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005269012093776837, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 869.7750244140625, "completions/mean_terminated_length": 583.357177734375, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.12576682679355145, "epoch": 0.8896473265073948, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013365321792662144, "kl": 0.012098742881789804, "learning_rate": 9.982935153583618e-07, "loss": 0.0554, "num_tokens": 164902047.0, "reward": 0.574999988079071, "reward_std": 0.8439072966575623, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.4318954944610596, "sampling/importance_sampling_ratio/mean": 0.35306185483932495, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7035343647003174, "sampling/sampling_logp_difference/mean": 0.015321852639317513, "step": 1564, "step_time": 72.36789001058787 }, { "clip_ratio/high_max": 0.01002860043081455, "clip_ratio/high_mean": 0.0025973184674512595, "clip_ratio/low_mean": 0.0011718750083673513, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003769193426705897, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 820.7999877929688, "completions/mean_terminated_length": 617.6000366210938, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.06418309127911925, "epoch": 0.8902161547212741, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024972479790449142, "kl": 0.00812511780532077, "learning_rate": 9.931740614334472e-07, "loss": 0.0097, "num_tokens": 164993543.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.8121771812438965, "sampling/importance_sampling_ratio/mean": 0.6081530451774597, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.14823579788208, "sampling/sampling_logp_difference/mean": 0.009343834593892097, "step": 1565, "step_time": 69.54194194450974 }, { "clip_ratio/high_max": 0.010022310714703053, "clip_ratio/high_mean": 0.0027841563569381833, "clip_ratio/low_mean": 0.001611328130820766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00439548451686278, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 878.1500244140625, "completions/mean_terminated_length": 758.8181762695312, "completions/min_length": 402.0, "completions/min_terminated_length": 402.0, "entropy": 0.055102610029280186, "epoch": 0.8907849829351536, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01660977490246296, "kl": 0.012167859822511673, "learning_rate": 9.880546075085324e-07, "loss": -0.014, "num_tokens": 165087277.0, "reward": 1.100000023841858, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.0643560886383057, "sampling/importance_sampling_ratio/mean": 0.3957025110721588, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4621007442474365, "sampling/sampling_logp_difference/mean": 0.00892956554889679, "step": 1566, "step_time": 72.9186563482508 }, { "clip_ratio/high_max": 0.012366694048978388, "clip_ratio/high_mean": 0.0028833550168201327, "clip_ratio/low_mean": 0.000561523440410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003444878471782431, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 959.0, "completions/mean_length": 820.4249877929688, "completions/mean_terminated_length": 616.8500366210938, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.11819731257855892, "epoch": 0.891353811149033, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.019155800342559814, "kl": 0.012529964849818498, "learning_rate": 9.829351535836179e-07, "loss": 0.0355, "num_tokens": 165195582.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.937084674835205, "sampling/importance_sampling_ratio/mean": 0.5431417226791382, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7742687463760376, "sampling/sampling_logp_difference/mean": 0.014582551084458828, "step": 1567, "step_time": 73.9606315405108 }, { "clip_ratio/high_max": 0.008868926437571645, "clip_ratio/high_mean": 0.0022170115844346583, "clip_ratio/low_mean": 0.00224609376891749, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004463105200557038, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 915.0, "completions/mean_length": 954.9750366210938, "completions/mean_terminated_length": 629.5714721679688, "completions/min_length": 433.0, "completions/min_terminated_length": 433.0, "entropy": 0.12813318241387606, "epoch": 0.8919226393629124, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.010247481986880302, "kl": 0.010963049891870469, "learning_rate": 9.77815699658703e-07, "loss": 0.0376, "num_tokens": 165304333.0, "reward": 0.30000001192092896, "reward_std": 0.6868732571601868, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 1.5915923118591309, "sampling/importance_sampling_ratio/mean": 0.32267704606056213, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.975965976715088, "sampling/sampling_logp_difference/mean": 0.013777839951217175, "step": 1568, "step_time": 75.05782878980972 }, { "clip_ratio/high_max": 0.014515428862068802, "clip_ratio/high_mean": 0.005126095289597288, "clip_ratio/low_mean": 0.001416015618815436, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006542110902955756, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 889.0, "completions/mean_length": 827.2000122070312, "completions/mean_terminated_length": 560.941162109375, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.10677228588610888, "epoch": 0.8924914675767918, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022128984332084656, "kl": 0.02839508594479412, "learning_rate": 9.726962457337885e-07, "loss": -0.035, "num_tokens": 165415349.0, "reward": 0.675000011920929, "reward_std": 0.8589648008346558, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.987318277359009, "sampling/importance_sampling_ratio/mean": 0.34458792209625244, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2540316581726074, "sampling/sampling_logp_difference/mean": 0.014220893383026123, "step": 1569, "step_time": 105.63490704796277 }, { "clip_ratio/high_max": 0.012476072297431529, "clip_ratio/high_mean": 0.0035797457094304264, "clip_ratio/low_mean": 0.0018310546583961695, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005410800178651698, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 930.0, "completions/mean_length": 931.2250366210938, "completions/mean_terminated_length": 652.9000244140625, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.11444913316518068, "epoch": 0.8930602957906713, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02071642316877842, "kl": 0.016765714448411018, "learning_rate": 9.675767918088737e-07, "loss": -0.002, "num_tokens": 165513470.0, "reward": 0.4749999940395355, "reward_std": 0.8469401597976685, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 1.0231022834777832, "sampling/importance_sampling_ratio/mean": 0.19294799864292145, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.324936866760254, "sampling/sampling_logp_difference/mean": 0.013009617105126381, "step": 1570, "step_time": 71.05115950130858 }, { "clip_ratio/high_max": 0.011687983467709273, "clip_ratio/high_mean": 0.0037060054892208427, "clip_ratio/low_mean": 0.002578208011982497, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006284213537583128, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 834.4500122070312, "completions/mean_terminated_length": 624.9473876953125, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.1372776059433818, "epoch": 0.8936291240045506, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01755535788834095, "kl": 0.018249048036523163, "learning_rate": 9.624573378839592e-07, "loss": 0.0015, "num_tokens": 165617480.0, "reward": 0.824999988079071, "reward_std": 0.957761824131012, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.9676201343536377, "sampling/importance_sampling_ratio/mean": 0.2249160259962082, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.599919319152832, "sampling/sampling_logp_difference/mean": 0.015735911205410957, "step": 1571, "step_time": 73.52939159655944 }, { "clip_ratio/high_max": 0.010986283887177706, "clip_ratio/high_mean": 0.0026069603045471013, "clip_ratio/low_mean": 0.0013183593837311491, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003925319746485911, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 789.9000244140625, "completions/mean_terminated_length": 633.8333740234375, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.14127783104777336, "epoch": 0.89419795221843, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022753093391656876, "kl": 0.012953967554494739, "learning_rate": 9.573378839590444e-07, "loss": -0.0744, "num_tokens": 165729108.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.489572525024414, "sampling/importance_sampling_ratio/mean": 0.41891103982925415, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0026397705078125, "sampling/sampling_logp_difference/mean": 0.01729867421090603, "step": 1572, "step_time": 74.70895074936561 }, { "clip_ratio/high_max": 0.01540531252976507, "clip_ratio/high_mean": 0.004672711918829009, "clip_ratio/low_mean": 0.002734374997089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0074070870177820325, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 903.0, "completions/mean_length": 832.7000122070312, "completions/mean_terminated_length": 621.26318359375, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.14221974276006222, "epoch": 0.8947667804323094, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.013891755603253841, "kl": 0.014643663773313165, "learning_rate": 9.522184300341297e-07, "loss": -0.022, "num_tokens": 165834800.0, "reward": 0.75, "reward_std": 0.8397191166877747, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.7380722761154175, "sampling/importance_sampling_ratio/mean": 0.20973923802375793, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1608383655548096, "sampling/sampling_logp_difference/mean": 0.015903113409876823, "step": 1573, "step_time": 74.52077563898638 }, { "clip_ratio/high_max": 0.013562424283009022, "clip_ratio/high_mean": 0.004864090449700598, "clip_ratio/low_mean": 0.002320217157830484, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0071843076148070395, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 837.8500366210938, "completions/mean_terminated_length": 632.1052856445312, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.16587443463504314, "epoch": 0.8953356086461889, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027384266257286072, "kl": 0.012733821524307132, "learning_rate": 9.470989761092152e-07, "loss": 0.0729, "num_tokens": 165930730.0, "reward": 0.875, "reward_std": 0.9657599329948425, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.0456790924072266, "sampling/importance_sampling_ratio/mean": 0.3029862344264984, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8812837600708008, "sampling/sampling_logp_difference/mean": 0.017373492941260338, "step": 1574, "step_time": 71.45228602434509 }, { "clip_ratio/high_max": 0.009451535763218999, "clip_ratio/high_mean": 0.002466807622113265, "clip_ratio/low_mean": 0.0017689836822682992, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004235791246173903, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 777.2999877929688, "completions/mean_terminated_length": 612.8333740234375, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.08812075294554234, "epoch": 0.8959044368600683, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023977547883987427, "kl": 0.011783790250774473, "learning_rate": 9.419795221843003e-07, "loss": -0.0419, "num_tokens": 166025278.0, "reward": 1.149999976158142, "reward_std": 0.9753369688987732, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.9028656482696533, "sampling/importance_sampling_ratio/mean": 0.4145265221595764, "sampling/importance_sampling_ratio/min": 0.00880709383636713, "sampling/sampling_logp_difference/max": 2.1511693000793457, "sampling/sampling_logp_difference/mean": 0.011386754922568798, "step": 1575, "step_time": 70.78980013704859 }, { "clip_ratio/high_max": 0.01318153494503349, "clip_ratio/high_mean": 0.0033671464916551486, "clip_ratio/low_mean": 0.0016113281290017767, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00497847463702783, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 807.0, "completions/mean_length": 802.6500244140625, "completions/mean_terminated_length": 558.0, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.10329880146309733, "epoch": 0.8964732650739476, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025532180443406105, "kl": 0.01419806998455897, "learning_rate": 9.368600682593856e-07, "loss": -0.0652, "num_tokens": 166132024.0, "reward": 0.824999988079071, "reward_std": 0.9306049346923828, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.849498987197876, "sampling/importance_sampling_ratio/mean": 0.3407270014286041, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1726632118225098, "sampling/sampling_logp_difference/mean": 0.012563707306981087, "step": 1576, "step_time": 73.311203192221 }, { "clip_ratio/high_max": 0.006003762420732528, "clip_ratio/high_mean": 0.0013291221985127777, "clip_ratio/low_mean": 0.0013046113235759549, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026337334857089445, "completions/clipped_ratio": 0.17499999701976776, "completions/max_length": 1024.0, "completions/max_terminated_length": 812.0, "completions/mean_length": 642.2000122070312, "completions/mean_terminated_length": 561.212158203125, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.10407190956175327, "epoch": 0.897042093287827, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.021566472947597504, "kl": 0.01662151759956032, "learning_rate": 9.317406143344709e-07, "loss": -0.0317, "num_tokens": 166239160.0, "reward": 1.5750000476837158, "reward_std": 0.7807787656784058, "rewards/format_reward/mean": 0.824999988079071, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.75, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.084636926651001, "sampling/importance_sampling_ratio/mean": 0.38727569580078125, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.881279468536377, "sampling/sampling_logp_difference/mean": 0.01520751416683197, "step": 1577, "step_time": 71.21366248442791 }, { "clip_ratio/high_max": 0.010332335834391415, "clip_ratio/high_mean": 0.0027028745535062626, "clip_ratio/low_mean": 0.0033260117925237864, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.006028886476997286, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 850.0, "completions/mean_length": 815.8250122070312, "completions/mean_terminated_length": 561.388916015625, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.0925401458516717, "epoch": 0.8976109215017065, "frac_reward_zero_std": 0.0, "grad_norm": 0.037661027163267136, "kl": 0.010599511209875345, "learning_rate": 9.266211604095563e-07, "loss": -0.135, "num_tokens": 166335481.0, "reward": 0.7250000238418579, "reward_std": 0.8766924738883972, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.3974926471710205, "sampling/importance_sampling_ratio/mean": 0.31055697798728943, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7742688655853271, "sampling/sampling_logp_difference/mean": 0.012237313203513622, "step": 1578, "step_time": 73.03732794197276 }, { "clip_ratio/high_max": 0.007757657556794584, "clip_ratio/high_mean": 0.0016274040244752541, "clip_ratio/low_mean": 0.003466676047537476, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005094080086564645, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 931.0, "completions/mean_length": 899.0499877929688, "completions/mean_terminated_length": 667.0, "completions/min_length": 474.0, "completions/min_terminated_length": 474.0, "entropy": 0.1268281163647771, "epoch": 0.8981797497155859, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02259557694196701, "kl": 0.010485222737770528, "learning_rate": 9.215017064846416e-07, "loss": 0.0726, "num_tokens": 166435259.0, "reward": 0.5, "reward_std": 0.751067578792572, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.1112678050994873, "sampling/importance_sampling_ratio/mean": 0.37782955169677734, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9272409677505493, "sampling/sampling_logp_difference/mean": 0.013804112561047077, "step": 1579, "step_time": 75.1696091638878 }, { "clip_ratio/high_max": 0.011575858807191253, "clip_ratio/high_mean": 0.002667288383236155, "clip_ratio/low_mean": 0.0018310546656721272, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004498342954320833, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 935.0, "completions/mean_length": 973.8250122070312, "completions/mean_terminated_length": 773.125, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.08658005576580763, "epoch": 0.8987485779294653, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.015682773664593697, "kl": 0.014306343160569668, "learning_rate": 9.16382252559727e-07, "loss": 0.0258, "num_tokens": 166549140.0, "reward": 0.4000000059604645, "reward_std": 0.8101914525032043, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 1.560284972190857, "sampling/importance_sampling_ratio/mean": 0.32977694272994995, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.5538177490234375, "sampling/sampling_logp_difference/mean": 0.011519104242324829, "step": 1580, "step_time": 76.6370533828158 }, { "clip_ratio/high_max": 0.005583551246672869, "clip_ratio/high_mean": 0.0011167102493345737, "clip_ratio/low_mean": 0.0007568359433207661, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018735461744654458, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 854.0, "completions/mean_length": 895.9000244140625, "completions/mean_terminated_length": 597.0, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.08183860033750534, "epoch": 0.8993174061433447, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.004461847245693207, "kl": 0.009719587513245642, "learning_rate": 9.112627986348123e-07, "loss": 0.0251, "num_tokens": 166656888.0, "reward": 0.6000000238418579, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.8235464096069336, "sampling/importance_sampling_ratio/mean": 0.3431882858276367, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8298583030700684, "sampling/sampling_logp_difference/mean": 0.010863730683922768, "step": 1581, "step_time": 73.70601170673035 }, { "clip_ratio/high_max": 0.010982719948515296, "clip_ratio/high_mean": 0.004627330257790163, "clip_ratio/low_mean": 0.002612304662761744, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.007239634956931695, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 877.1500244140625, "completions/mean_terminated_length": 678.4705810546875, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.1343136327341199, "epoch": 0.8998862343572241, "frac_reward_zero_std": 0.0, "grad_norm": 0.033385828137397766, "kl": 0.01298888970632106, "learning_rate": 9.061433447098977e-07, "loss": 0.0325, "num_tokens": 166765390.0, "reward": 0.800000011920929, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.7305058240890503, "sampling/importance_sampling_ratio/mean": 0.4331475794315338, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9450607299804688, "sampling/sampling_logp_difference/mean": 0.01470195036381483, "step": 1582, "step_time": 81.62219410086982 }, { "clip_ratio/high_max": 0.01376722112763673, "clip_ratio/high_mean": 0.005761321648606099, "clip_ratio/low_mean": 0.002661132784851361, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00842245452804491, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 866.7000122070312, "completions/mean_terminated_length": 738.0, "completions/min_length": 435.0, "completions/min_terminated_length": 435.0, "entropy": 0.18036215752363205, "epoch": 0.9004550625711035, "frac_reward_zero_std": 0.0, "grad_norm": 0.054881032556295395, "kl": 0.012590479804202914, "learning_rate": 9.01023890784983e-07, "loss": -0.1922, "num_tokens": 166876042.0, "reward": 0.925000011920929, "reward_std": 0.9167249202728271, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.9197182655334473, "sampling/importance_sampling_ratio/mean": 0.605836033821106, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.313965082168579, "sampling/sampling_logp_difference/mean": 0.01791423000395298, "step": 1583, "step_time": 80.11691635474563 }, { "clip_ratio/high_max": 0.013801735127344728, "clip_ratio/high_mean": 0.004012403267552145, "clip_ratio/low_mean": 0.0019775390337599674, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005989942408632487, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 892.0, "completions/mean_length": 881.1749877929688, "completions/mean_terminated_length": 584.5385131835938, "completions/min_length": 401.0, "completions/min_terminated_length": 401.0, "entropy": 0.11688633635640144, "epoch": 0.9010238907849829, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02601195126771927, "kl": 0.012268297839909792, "learning_rate": 8.959044368600683e-07, "loss": -0.0619, "num_tokens": 166979497.0, "reward": 0.45000001788139343, "reward_std": 0.7143223285675049, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.8539719581604004, "sampling/importance_sampling_ratio/mean": 0.5603101849555969, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.392754077911377, "sampling/sampling_logp_difference/mean": 0.014109528623521328, "step": 1584, "step_time": 77.00596054294147 }, { "clip_ratio/high_max": 0.008644320827443153, "clip_ratio/high_mean": 0.0023733795678708702, "clip_ratio/low_mean": 0.0017333984396827873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004106777982087806, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 979.0, "completions/mean_length": 955.2000122070312, "completions/mean_terminated_length": 680.0, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "entropy": 0.10304463095963001, "epoch": 0.9015927189988624, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.021535616368055344, "kl": 0.011397580965422094, "learning_rate": 8.907849829351537e-07, "loss": 0.0192, "num_tokens": 167094649.0, "reward": 0.3499999940395355, "reward_std": 0.7355444431304932, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.4767837524414062, "sampling/importance_sampling_ratio/mean": 0.37611547112464905, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8812990188598633, "sampling/sampling_logp_difference/mean": 0.012782362289726734, "step": 1585, "step_time": 77.69533145823516 }, { "clip_ratio/high_max": 0.016206614673137665, "clip_ratio/high_mean": 0.005110847385367379, "clip_ratio/low_mean": 0.0019775390828726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007088386511895806, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 870.5, "completions/mean_terminated_length": 682.888916015625, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.09383661951869726, "epoch": 0.9021615472127418, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.039950836449861526, "kl": 0.009617018688004464, "learning_rate": 8.85665529010239e-07, "loss": -0.0625, "num_tokens": 167198797.0, "reward": 0.7750000357627869, "reward_std": 0.9195176959037781, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.8576793670654297, "sampling/importance_sampling_ratio/mean": 0.5145465135574341, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1961774826049805, "sampling/sampling_logp_difference/mean": 0.012020071037113667, "step": 1586, "step_time": 78.68265875591896 }, { "clip_ratio/high_max": 0.01491109689231962, "clip_ratio/high_mean": 0.006405879306839779, "clip_ratio/low_mean": 0.0009958542796084657, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00740173360100016, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 779.6000366210938, "completions/mean_terminated_length": 579.6363525390625, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.10874996241182089, "epoch": 0.9027303754266212, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.056839797645807266, "kl": 0.010694912285543978, "learning_rate": 8.805460750853243e-07, "loss": 0.0838, "num_tokens": 167294477.0, "reward": 1.0750000476837158, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.647998094558716, "sampling/importance_sampling_ratio/mean": 0.5566843748092651, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3766937255859375, "sampling/sampling_logp_difference/mean": 0.013791384175419807, "step": 1587, "step_time": 71.7409798828885 }, { "clip_ratio/high_max": 0.008108737471047789, "clip_ratio/high_mean": 0.0023289028631552355, "clip_ratio/low_mean": 0.0022460937761934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00457499671028927, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 903.0, "completions/mean_length": 922.5250244140625, "completions/mean_terminated_length": 655.0, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.0942654088139534, "epoch": 0.9032992036405005, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02721838466823101, "kl": 0.010807299753651023, "learning_rate": 8.754266211604096e-07, "loss": 0.1219, "num_tokens": 167405250.0, "reward": 0.5250000357627869, "reward_std": 0.8469401597976685, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.9246490001678467, "sampling/importance_sampling_ratio/mean": 0.5566462874412537, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5158462524414062, "sampling/sampling_logp_difference/mean": 0.011533289216458797, "step": 1588, "step_time": 76.46787889976986 }, { "clip_ratio/high_max": 0.01740250934381038, "clip_ratio/high_mean": 0.008265281619969755, "clip_ratio/low_mean": 0.0013890377013012767, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009654319321271032, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 703.9249877929688, "completions/mean_terminated_length": 511.8799743652344, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.1450233180075884, "epoch": 0.90386803185438, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.011468975804746151, "kl": 0.013884419342502952, "learning_rate": 8.70307167235495e-07, "loss": 0.0063, "num_tokens": 167509415.0, "reward": 1.2000000476837158, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.0899276733398438, "sampling/importance_sampling_ratio/mean": 0.3430103063583374, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.986206293106079, "sampling/sampling_logp_difference/mean": 0.017626119777560234, "step": 1589, "step_time": 72.83463621954434 }, { "clip_ratio/high_max": 0.015728165628388524, "clip_ratio/high_mean": 0.00500629234011285, "clip_ratio/low_mean": 0.002392578113358468, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007398870657198131, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 977.0, "completions/mean_length": 778.2999877929688, "completions/mean_terminated_length": 614.5, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.13373701367527246, "epoch": 0.9044368600682594, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.043921735137701035, "kl": 0.01632283127401024, "learning_rate": 8.651877133105803e-07, "loss": 0.0069, "num_tokens": 167607299.0, "reward": 1.1750000715255737, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.955620527267456, "sampling/importance_sampling_ratio/mean": 0.6143477559089661, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5255517959594727, "sampling/sampling_logp_difference/mean": 0.015714410692453384, "step": 1590, "step_time": 69.96398854651488 }, { "clip_ratio/high_max": 0.0116622353089042, "clip_ratio/high_mean": 0.0031430609960807487, "clip_ratio/low_mean": 0.002520847920095548, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005663909076247364, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 761.4000244140625, "completions/mean_terminated_length": 634.9629516601562, "completions/min_length": 483.0, "completions/min_terminated_length": 483.0, "entropy": 0.09791779052466154, "epoch": 0.9050056882821388, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.042547523975372314, "kl": 0.010525953315664083, "learning_rate": 8.600682593856656e-07, "loss": -0.0749, "num_tokens": 167705603.0, "reward": 1.0500000715255737, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.5229337215423584, "sampling/importance_sampling_ratio/mean": 0.46461448073387146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.7841989994049072, "sampling/sampling_logp_difference/mean": 0.012702936306595802, "step": 1591, "step_time": 73.85094400821254 }, { "clip_ratio/high_max": 0.013250764110125601, "clip_ratio/high_mean": 0.004285118789994158, "clip_ratio/low_mean": 0.0016845702702994458, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0059696890821214765, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 927.0, "completions/mean_length": 667.6000366210938, "completions/mean_terminated_length": 532.413818359375, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.137205527164042, "epoch": 0.9055745164960182, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.023674361407756805, "kl": 0.015581880346871912, "learning_rate": 8.549488054607508e-07, "loss": -0.0763, "num_tokens": 167806179.0, "reward": 1.3250000476837158, "reward_std": 0.888314425945282, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.924321174621582, "sampling/importance_sampling_ratio/mean": 0.4276486337184906, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3986610174179077, "sampling/sampling_logp_difference/mean": 0.01800396852195263, "step": 1592, "step_time": 71.88930125907063 }, { "clip_ratio/high_max": 0.017270736279897392, "clip_ratio/high_mean": 0.006934638338861987, "clip_ratio/low_mean": 0.0005126953255967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007447333657182753, "completions/clipped_ratio": 0.125, "completions/max_length": 1024.0, "completions/max_terminated_length": 837.0, "completions/mean_length": 619.1749877929688, "completions/mean_terminated_length": 561.3428344726562, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.13772432692348957, "epoch": 0.9061433447098977, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022947141900658607, "kl": 0.01675470918416977, "learning_rate": 8.498293515358362e-07, "loss": -0.0504, "num_tokens": 167900898.0, "reward": 1.5500000715255737, "reward_std": 0.7143223285675049, "rewards/format_reward/mean": 0.875, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.7089645862579346, "sampling/importance_sampling_ratio/mean": 0.43846994638442993, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.562098264694214, "sampling/sampling_logp_difference/mean": 0.018107131123542786, "step": 1593, "step_time": 72.67246314580552 }, { "clip_ratio/high_max": 0.011871678580064327, "clip_ratio/high_mean": 0.002999145755893551, "clip_ratio/low_mean": 0.003051757834327873, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006050903524737805, "completions/clipped_ratio": 0.800000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 853.0, "completions/mean_length": 941.3500366210938, "completions/mean_terminated_length": 610.75, "completions/min_length": 442.0, "completions/min_terminated_length": 442.0, "entropy": 0.09412385895848274, "epoch": 0.906712172923777, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022316884249448776, "kl": 0.010494650166947395, "learning_rate": 8.447098976109215e-07, "loss": 0.0093, "num_tokens": 168013112.0, "reward": 0.4000000059604645, "reward_std": 0.8101915717124939, "rewards/format_reward/mean": 0.20000000298023224, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.229715347290039, "sampling/importance_sampling_ratio/mean": 0.36494722962379456, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.271134376525879, "sampling/sampling_logp_difference/mean": 0.011839902959764004, "step": 1594, "step_time": 76.75075346883386 }, { "clip_ratio/high_max": 0.014454394578933716, "clip_ratio/high_mean": 0.005038013652665541, "clip_ratio/low_mean": 0.0033512994705233723, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008389313123188913, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 929.0, "completions/mean_length": 807.9249877929688, "completions/mean_terminated_length": 631.1363525390625, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.1505830604583025, "epoch": 0.9072810011376564, "frac_reward_zero_std": 0.0, "grad_norm": 0.025988714769482613, "kl": 0.01696554117370397, "learning_rate": 8.395904436860068e-07, "loss": -0.0213, "num_tokens": 168118045.0, "reward": 0.9750000238418579, "reward_std": 0.9469925761222839, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.3929898738861084, "sampling/importance_sampling_ratio/mean": 0.44043463468551636, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.554166316986084, "sampling/sampling_logp_difference/mean": 0.016715075820684433, "step": 1595, "step_time": 71.2639022199437 }, { "clip_ratio/high_max": 0.011688337661325932, "clip_ratio/high_mean": 0.0035578617098508403, "clip_ratio/low_mean": 0.0018798828241415322, "clip_ratio/low_min": 0.0003662109375, "clip_ratio/region_mean": 0.005437744548544288, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.0, "completions/mean_length": 877.0250244140625, "completions/mean_terminated_length": 604.0714721679688, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 0.0551592749543488, "epoch": 0.9078498293515358, "frac_reward_zero_std": 0.0, "grad_norm": 0.026202239096164703, "kl": 0.01083237468264997, "learning_rate": 8.344709897610922e-07, "loss": -0.0232, "num_tokens": 168228366.0, "reward": 0.699999988079071, "reward_std": 0.9660917520523071, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.5405609607696533, "sampling/importance_sampling_ratio/mean": 0.28095144033432007, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9008355140686035, "sampling/sampling_logp_difference/mean": 0.009285620413720608, "step": 1596, "step_time": 75.97111312160268 }, { "clip_ratio/high_max": 0.00796199106844142, "clip_ratio/high_mean": 0.0020452640019357204, "clip_ratio/low_mean": 0.002905273431679234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004950537579134107, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 852.3250122070312, "completions/mean_terminated_length": 594.8125, "completions/min_length": 381.0, "completions/min_terminated_length": 381.0, "entropy": 0.11148878652602434, "epoch": 0.9084186575654153, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02509300969541073, "kl": 0.011854388401843607, "learning_rate": 8.293515358361775e-07, "loss": 0.0683, "num_tokens": 168334835.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.159341335296631, "sampling/importance_sampling_ratio/mean": 0.24958333373069763, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.218155860900879, "sampling/sampling_logp_difference/mean": 0.014148922637104988, "step": 1597, "step_time": 74.76486858539283 }, { "clip_ratio/high_max": 0.010046396520920098, "clip_ratio/high_mean": 0.0028045308135915548, "clip_ratio/low_mean": 0.0024414062791038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005245937063591555, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 885.6749877929688, "completions/mean_terminated_length": 470.70001220703125, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.080920385196805, "epoch": 0.9089874857792947, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03023618459701538, "kl": 0.01120058581000194, "learning_rate": 8.242320819112628e-07, "loss": -0.0487, "num_tokens": 168447422.0, "reward": 0.4749999940395355, "reward_std": 0.8469402194023132, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290207743644714, "sampling/importance_sampling_ratio/max": 2.8050379753112793, "sampling/importance_sampling_ratio/mean": 0.48704108595848083, "sampling/importance_sampling_ratio/min": 0.006079448387026787, "sampling/sampling_logp_difference/max": 2.847175121307373, "sampling/sampling_logp_difference/mean": 0.01106905099004507, "step": 1598, "step_time": 75.44585295952857 }, { "clip_ratio/high_max": 0.012834636261686683, "clip_ratio/high_mean": 0.003858050229609944, "clip_ratio/low_mean": 0.0013916015705035534, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005249651672784239, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 894.0, "completions/mean_length": 693.7250366210938, "completions/mean_terminated_length": 583.6333618164062, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.14890972524881363, "epoch": 0.909556313993174, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.017452402040362358, "kl": 0.013847352820448577, "learning_rate": 8.191126279863481e-07, "loss": -0.034, "num_tokens": 168545939.0, "reward": 1.5, "reward_std": 0.8770580291748047, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.75, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.3389673233032227, "sampling/importance_sampling_ratio/mean": 0.3542417585849762, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6851205825805664, "sampling/sampling_logp_difference/mean": 0.018836162984371185, "step": 1599, "step_time": 71.91723481426015 }, { "clip_ratio/high_max": 0.013644080318044871, "clip_ratio/high_mean": 0.004049799244967289, "clip_ratio/low_mean": 0.0024414062645519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006491205538623035, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 974.0, "completions/mean_length": 932.7750244140625, "completions/mean_terminated_length": 743.3077392578125, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.11633270885795355, "epoch": 0.9101251422070534, "frac_reward_zero_std": 0.0, "grad_norm": 0.024960439652204514, "kl": 0.011289743357338011, "learning_rate": 8.139931740614335e-07, "loss": -0.014, "num_tokens": 168652642.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.473508358001709, "sampling/importance_sampling_ratio/mean": 0.44110703468322754, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9192707538604736, "sampling/sampling_logp_difference/mean": 0.012583166360855103, "step": 1600, "step_time": 74.21558144316077 }, { "clip_ratio/high_max": 0.016295487992465496, "clip_ratio/high_mean": 0.004967753557139076, "clip_ratio/low_mean": 0.001806640655559022, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006774394074454904, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 745.4500122070312, "completions/mean_terminated_length": 539.5652465820312, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.12032747641205788, "epoch": 0.9106939704209329, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01786021515727043, "kl": 0.014839199255220592, "learning_rate": 8.088737201365188e-07, "loss": 0.0587, "num_tokens": 168745052.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.7925055027008057, "sampling/importance_sampling_ratio/mean": 0.36161237955093384, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0964255332946777, "sampling/sampling_logp_difference/mean": 0.014648035168647766, "step": 1601, "step_time": 70.91040984308347 }, { "clip_ratio/high_max": 0.01390054140938446, "clip_ratio/high_mean": 0.005827702203532681, "clip_ratio/low_mean": 0.0015238040796248242, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007351506385020912, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 899.0, "completions/mean_length": 765.1500244140625, "completions/mean_terminated_length": 573.8261108398438, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.12731157429516315, "epoch": 0.9112627986348123, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.037520937621593475, "kl": 0.014196795469615608, "learning_rate": 8.037542662116041e-07, "loss": 0.0534, "num_tokens": 168844778.0, "reward": 1.0750000476837158, "reward_std": 0.9710555076599121, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.923601508140564, "sampling/importance_sampling_ratio/mean": 0.3625714182853699, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5555613040924072, "sampling/sampling_logp_difference/mean": 0.015532451681792736, "step": 1602, "step_time": 77.49705989286304 }, { "clip_ratio/high_max": 0.0073958022985607386, "clip_ratio/high_mean": 0.002313961216714233, "clip_ratio/low_mean": 0.0017713793531584088, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004085340537130833, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 882.0, "completions/mean_length": 869.625, "completions/mean_terminated_length": 582.9285888671875, "completions/min_length": 402.0, "completions/min_terminated_length": 402.0, "entropy": 0.07997412094846368, "epoch": 0.9118316268486917, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023787321522831917, "kl": 0.009454419137910008, "learning_rate": 7.986348122866895e-07, "loss": -0.0732, "num_tokens": 168945947.0, "reward": 0.6000000238418579, "reward_std": 0.8711913228034973, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.7994321584701538, "sampling/importance_sampling_ratio/mean": 0.47820577025413513, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3167362213134766, "sampling/sampling_logp_difference/mean": 0.010181024670600891, "step": 1603, "step_time": 74.50274115102366 }, { "clip_ratio/high_max": 0.010592918668407947, "clip_ratio/high_mean": 0.003022470642463304, "clip_ratio/low_mean": 0.0018554687703726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004877939441939816, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 824.0, "completions/mean_length": 741.125, "completions/mean_terminated_length": 588.8077392578125, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.13002505339682102, "epoch": 0.9124004550625711, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04726165160536766, "kl": 0.012933194171637297, "learning_rate": 7.935153583617748e-07, "loss": -0.0266, "num_tokens": 169041200.0, "reward": 1.274999976158142, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.625, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.987447738647461, "sampling/importance_sampling_ratio/mean": 0.5087175369262695, "sampling/importance_sampling_ratio/min": 0.001409450895152986, "sampling/sampling_logp_difference/max": 2.318364381790161, "sampling/sampling_logp_difference/mean": 0.014947625808417797, "step": 1604, "step_time": 70.85214037704282 }, { "clip_ratio/high_max": 0.01400511502288282, "clip_ratio/high_mean": 0.003858845608192496, "clip_ratio/low_mean": 0.0016143094089784427, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005473155091749504, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 895.7250366210938, "completions/mean_terminated_length": 703.3125, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.13509217463433743, "epoch": 0.9129692832764505, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03170134872198105, "kl": 0.011271935072727501, "learning_rate": 7.883959044368601e-07, "loss": -0.0718, "num_tokens": 169150509.0, "reward": 0.675000011920929, "reward_std": 0.888314425945282, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.377161979675293, "sampling/importance_sampling_ratio/mean": 0.43164747953414917, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6036088466644287, "sampling/sampling_logp_difference/mean": 0.014709427952766418, "step": 1605, "step_time": 76.96826241561212 }, { "clip_ratio/high_max": 0.013983621669467539, "clip_ratio/high_mean": 0.0036761871306225657, "clip_ratio/low_mean": 0.00021972656941215973, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038959136290941387, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 732.1500244140625, "completions/mean_terminated_length": 440.3000183105469, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.1005927873775363, "epoch": 0.9135381114903299, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.014286589808762074, "kl": 0.010238006361760199, "learning_rate": 7.832764505119454e-07, "loss": -0.032, "num_tokens": 169252691.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.9049299955368042, "sampling/importance_sampling_ratio/mean": 0.5854738354682922, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.272998332977295, "sampling/sampling_logp_difference/mean": 0.012307542376220226, "step": 1606, "step_time": 71.7472885325551 }, { "clip_ratio/high_max": 0.011305150110274553, "clip_ratio/high_mean": 0.0036074441741220653, "clip_ratio/low_mean": 0.0024311626075359527, "clip_ratio/low_min": 0.0006103515625, "clip_ratio/region_mean": 0.006038606748916209, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 899.7750244140625, "completions/mean_terminated_length": 609.9166870117188, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.09182066190987825, "epoch": 0.9141069397042093, "frac_reward_zero_std": 0.0, "grad_norm": 0.02955351583659649, "kl": 0.011990990140475333, "learning_rate": 7.781569965870308e-07, "loss": -0.0272, "num_tokens": 169357778.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.1663050651550293, "sampling/importance_sampling_ratio/mean": 0.33328866958618164, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2362513542175293, "sampling/sampling_logp_difference/mean": 0.011596093885600567, "step": 1607, "step_time": 75.58502905443311 }, { "clip_ratio/high_max": 0.010041460060165264, "clip_ratio/high_mean": 0.0031399062754644547, "clip_ratio/low_mean": 0.0019236587395425886, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00506356505502481, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 961.0, "completions/mean_length": 817.5, "completions/mean_terminated_length": 648.5454711914062, "completions/min_length": 429.0, "completions/min_terminated_length": 429.0, "entropy": 0.08111094497144222, "epoch": 0.9146757679180887, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029423944652080536, "kl": 0.007781450403854251, "learning_rate": 7.730375426621161e-07, "loss": -0.0596, "num_tokens": 169461670.0, "reward": 0.875, "reward_std": 0.8825225830078125, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.6370155811309814, "sampling/importance_sampling_ratio/mean": 0.5857416391372681, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9013915061950684, "sampling/sampling_logp_difference/mean": 0.010839520953595638, "step": 1608, "step_time": 75.18568763346411 }, { "clip_ratio/high_max": 0.01142344536492601, "clip_ratio/high_mean": 0.003252809794503264, "clip_ratio/low_mean": 0.001977539053768851, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005230348790064454, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 956.0, "completions/mean_length": 955.4750366210938, "completions/mean_terminated_length": 774.8181762695312, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "entropy": 0.09603298269212246, "epoch": 0.9152445961319682, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.025379642844200134, "kl": 0.010611686448100954, "learning_rate": 7.679180887372014e-07, "loss": -0.082, "num_tokens": 169573321.0, "reward": 0.42500001192092896, "reward_std": 0.747217059135437, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.148289203643799, "sampling/importance_sampling_ratio/mean": 0.30059584975242615, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2730393409729004, "sampling/sampling_logp_difference/mean": 0.011191285215318203, "step": 1609, "step_time": 76.66798354010098 }, { "clip_ratio/high_max": 0.015716780675575137, "clip_ratio/high_mean": 0.004958918711054139, "clip_ratio/low_mean": 0.0026855468822759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007644465484190732, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 884.9750366210938, "completions/mean_terminated_length": 676.4375, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.14737372379750013, "epoch": 0.9158134243458476, "frac_reward_zero_std": 0.0, "grad_norm": 0.018473347648978233, "kl": 0.015720056951977313, "learning_rate": 7.627986348122866e-07, "loss": -0.0403, "num_tokens": 169680520.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.7249420881271362, "sampling/importance_sampling_ratio/mean": 0.2180200070142746, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0040907859802246, "sampling/sampling_logp_difference/mean": 0.015802530571818352, "step": 1610, "step_time": 74.24113781098276 }, { "clip_ratio/high_max": 0.011652071785647422, "clip_ratio/high_mean": 0.004591450793668628, "clip_ratio/low_mean": 0.0010009765865106601, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005592427478404716, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 815.8250122070312, "completions/mean_terminated_length": 561.388916015625, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "entropy": 0.0769564425572753, "epoch": 0.9163822525597269, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030247490853071213, "kl": 0.009525873232632875, "learning_rate": 7.57679180887372e-07, "loss": 0.0571, "num_tokens": 169778969.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.410761833190918, "sampling/importance_sampling_ratio/mean": 0.4013567566871643, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9289193153381348, "sampling/sampling_logp_difference/mean": 0.01043830718845129, "step": 1611, "step_time": 71.02535105962306 }, { "clip_ratio/high_max": 0.009727314696647227, "clip_ratio/high_mean": 0.003058408387005329, "clip_ratio/low_mean": 0.0021840754579898203, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005242483719484881, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 978.0, "completions/mean_length": 901.1749877929688, "completions/mean_terminated_length": 716.9375, "completions/min_length": 473.0, "completions/min_terminated_length": 473.0, "entropy": 0.10376440826803446, "epoch": 0.9169510807736063, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030982863157987595, "kl": 0.00909172318642959, "learning_rate": 7.525597269624573e-07, "loss": -0.0554, "num_tokens": 169875952.0, "reward": 0.675000011920929, "reward_std": 0.888314425945282, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.091088056564331, "sampling/importance_sampling_ratio/mean": 0.42495545744895935, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8305293321609497, "sampling/sampling_logp_difference/mean": 0.01236059982329607, "step": 1612, "step_time": 75.5677302617114 }, { "clip_ratio/high_max": 0.012120273313485086, "clip_ratio/high_mean": 0.0038180731353349984, "clip_ratio/low_mean": 0.0014413102217076812, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00525938332430087, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 875.0, "completions/mean_length": 864.6749877929688, "completions/mean_terminated_length": 625.6875, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.07905292697250843, "epoch": 0.9175199089874858, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01635017804801464, "kl": 0.009707676246762276, "learning_rate": 7.474402730375426e-07, "loss": 0.0225, "num_tokens": 169980995.0, "reward": 0.75, "reward_std": 0.9540735483169556, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.7638347148895264, "sampling/importance_sampling_ratio/mean": 0.3719720244407654, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.69578218460083, "sampling/sampling_logp_difference/mean": 0.01069826539605856, "step": 1613, "step_time": 73.57146219396964 }, { "clip_ratio/high_max": 0.013628351967781782, "clip_ratio/high_mean": 0.0053937111515551805, "clip_ratio/low_mean": 0.002783203097351361, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.008176914183422923, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 810.5750122070312, "completions/mean_terminated_length": 597.1500244140625, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.13704617973417044, "epoch": 0.9180887372013652, "frac_reward_zero_std": 0.0, "grad_norm": 0.032506465911865234, "kl": 0.024610100663267076, "learning_rate": 7.42320819112628e-07, "loss": 0.0621, "num_tokens": 170088770.0, "reward": 0.949999988079071, "reward_std": 0.9857965111732483, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.7717232704162598, "sampling/importance_sampling_ratio/mean": 0.46316638588905334, "sampling/importance_sampling_ratio/min": 0.0005702785565517843, "sampling/sampling_logp_difference/max": 2.9585494995117188, "sampling/sampling_logp_difference/mean": 0.015242086723446846, "step": 1614, "step_time": 74.30291760805994 }, { "clip_ratio/high_max": 0.007098921341821551, "clip_ratio/high_mean": 0.0014197842974681407, "clip_ratio/low_mean": 0.0005615234476863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019813077451544814, "completions/clipped_ratio": 0.125, "completions/max_length": 1024.0, "completions/max_terminated_length": 864.0, "completions/mean_length": 613.7000122070312, "completions/mean_terminated_length": 555.085693359375, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.10223126225173473, "epoch": 0.9186575654152446, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.010688326321542263, "kl": 0.01366638473700732, "learning_rate": 7.372013651877133e-07, "loss": -0.0166, "num_tokens": 170196974.0, "reward": 1.75, "reward_std": 0.6698641180992126, "rewards/format_reward/mean": 0.875, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.875, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.450481414794922, "sampling/importance_sampling_ratio/mean": 0.3693634867668152, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7682969570159912, "sampling/sampling_logp_difference/mean": 0.01578003168106079, "step": 1615, "step_time": 71.89020920894109 }, { "clip_ratio/high_max": 0.01395354449050501, "clip_ratio/high_mean": 0.004767949489178136, "clip_ratio/low_mean": 0.002332678181119263, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007100627524778247, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 905.0, "completions/mean_length": 736.9000244140625, "completions/mean_terminated_length": 502.0, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.08742905780673027, "epoch": 0.919226393629124, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0320398285984993, "kl": 0.013175573898479342, "learning_rate": 7.320819112627986e-07, "loss": 0.0203, "num_tokens": 170283778.0, "reward": 0.925000011920929, "reward_std": 0.9167249798774719, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.7998626232147217, "sampling/importance_sampling_ratio/mean": 0.531436562538147, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.527376174926758, "sampling/sampling_logp_difference/mean": 0.012502709403634071, "step": 1616, "step_time": 70.36302618379705 }, { "clip_ratio/high_max": 0.013614347786642611, "clip_ratio/high_mean": 0.004513523570494726, "clip_ratio/low_mean": 0.0020645381446229294, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006578061904292554, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 874.0, "completions/mean_length": 861.1000366210938, "completions/mean_terminated_length": 640.7058715820312, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.11191750504076481, "epoch": 0.9197952218430034, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03907933086156845, "kl": 0.012136236182413995, "learning_rate": 7.269624573378839e-07, "loss": -0.0414, "num_tokens": 170388302.0, "reward": 0.75, "reward_std": 0.9268086552619934, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.6382644176483154, "sampling/importance_sampling_ratio/mean": 0.5352932810783386, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7621002197265625, "sampling/sampling_logp_difference/mean": 0.013570397160947323, "step": 1617, "step_time": 72.3535139192827 }, { "clip_ratio/high_max": 0.017361157573759556, "clip_ratio/high_mean": 0.005924933328060433, "clip_ratio/low_mean": 0.000866833383042831, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006791766558308154, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 680.0, "completions/mean_length": 711.1749877929688, "completions/mean_terminated_length": 398.3500061035156, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.0850512720644474, "epoch": 0.9203640500568828, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.014411695301532745, "kl": 0.010648232884705067, "learning_rate": 7.218430034129693e-07, "loss": -0.004, "num_tokens": 170492933.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.4225051403045654, "sampling/importance_sampling_ratio/mean": 0.4016265869140625, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.346745491027832, "sampling/sampling_logp_difference/mean": 0.011999205686151981, "step": 1618, "step_time": 73.59708526893519 }, { "clip_ratio/high_max": 0.012431867886334658, "clip_ratio/high_mean": 0.005444362643174827, "clip_ratio/low_mean": 0.0020321741249063052, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007476536789909005, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 710.5499877929688, "completions/mean_terminated_length": 478.86956787109375, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.1202841280028224, "epoch": 0.9209328782707622, "frac_reward_zero_std": 0.0, "grad_norm": 0.024131326004862785, "kl": 0.01943327847402543, "learning_rate": 7.167235494880546e-07, "loss": -0.019, "num_tokens": 170600571.0, "reward": 1.100000023841858, "reward_std": 0.9818872213363647, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.9274795055389404, "sampling/importance_sampling_ratio/mean": 0.31271180510520935, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9722328186035156, "sampling/sampling_logp_difference/mean": 0.015143233351409435, "step": 1619, "step_time": 77.36722692102194 }, { "clip_ratio/high_max": 0.01522440480766818, "clip_ratio/high_mean": 0.005405920717748813, "clip_ratio/low_mean": 0.0015751684877614025, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006981089187320322, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 830.6749877929688, "completions/mean_terminated_length": 594.388916015625, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.11512102466076612, "epoch": 0.9215017064846417, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.027773329988121986, "kl": 0.012842284282669425, "learning_rate": 7.116040955631399e-07, "loss": -0.0688, "num_tokens": 170703398.0, "reward": 0.8500000238418579, "reward_std": 0.9753369092941284, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.6176154613494873, "sampling/importance_sampling_ratio/mean": 0.4115125238895416, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.473586082458496, "sampling/sampling_logp_difference/mean": 0.013966833241283894, "step": 1620, "step_time": 79.50815872382373 }, { "clip_ratio/high_max": 0.013461084221489727, "clip_ratio/high_mean": 0.005539271107409149, "clip_ratio/low_mean": 0.0010742187751020538, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006613489822484553, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 804.875, "completions/mean_terminated_length": 606.6190795898438, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.1142010223120451, "epoch": 0.9220705346985211, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0419086217880249, "kl": 0.03660179430153221, "learning_rate": 7.064846416382253e-07, "loss": 0.0861, "num_tokens": 170813425.0, "reward": 0.925000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057363510131836, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.6556808948516846, "sampling/importance_sampling_ratio/mean": 0.3927481770515442, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.12725567817688, "sampling/sampling_logp_difference/mean": 0.014781191945075989, "step": 1621, "step_time": 74.39186203991994 }, { "clip_ratio/high_max": 0.010862810479011387, "clip_ratio/high_mean": 0.0037686580326408148, "clip_ratio/low_mean": 0.0006835937601863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004452251770999283, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 900.9500122070312, "completions/mean_terminated_length": 750.5555419921875, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.05430947337299585, "epoch": 0.9226393629124005, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01814189925789833, "kl": 0.0095868207863532, "learning_rate": 7.013651877133106e-07, "loss": -0.0298, "num_tokens": 170915631.0, "reward": 0.75, "reward_std": 0.8987170457839966, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.8795318603515625, "sampling/importance_sampling_ratio/mean": 0.44235095381736755, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.574509859085083, "sampling/sampling_logp_difference/mean": 0.008716721087694168, "step": 1622, "step_time": 73.20574268512428 }, { "clip_ratio/high_max": 0.01372767030261457, "clip_ratio/high_mean": 0.005542270751902834, "clip_ratio/low_mean": 0.0010360122123529436, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006578282918781042, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 760.9500122070312, "completions/mean_terminated_length": 619.3077392578125, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.07763809710741043, "epoch": 0.9232081911262798, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.032198358327150345, "kl": 0.014228462125174701, "learning_rate": 6.962457337883959e-07, "loss": 0.0414, "num_tokens": 171022341.0, "reward": 1.225000023841858, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.9952666759490967, "sampling/importance_sampling_ratio/mean": 0.42985764145851135, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.600800037384033, "sampling/sampling_logp_difference/mean": 0.012805468402802944, "step": 1623, "step_time": 73.22365209134296 }, { "clip_ratio/high_max": 0.015878278180025518, "clip_ratio/high_mean": 0.005226457171374932, "clip_ratio/low_mean": 0.001928710909851361, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007155167986638844, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 840.2750244140625, "completions/mean_terminated_length": 656.5499877929688, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.0972233209758997, "epoch": 0.9237770193401593, "frac_reward_zero_std": 0.0, "grad_norm": 0.031523045152425766, "kl": 0.011652292916551232, "learning_rate": 6.911262798634812e-07, "loss": -0.0802, "num_tokens": 171122008.0, "reward": 0.949999988079071, "reward_std": 0.9857965111732483, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.050110101699829, "sampling/importance_sampling_ratio/mean": 0.4345242977142334, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0708024501800537, "sampling/sampling_logp_difference/mean": 0.012673253193497658, "step": 1624, "step_time": 74.30058782966807 }, { "clip_ratio/high_max": 0.015053226263262331, "clip_ratio/high_mean": 0.0053898823680356145, "clip_ratio/low_mean": 0.001928710931679234, "clip_ratio/low_min": 0.0001220703125, "clip_ratio/region_mean": 0.007318593154195696, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 881.1749877929688, "completions/mean_terminated_length": 687.941162109375, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.11151161696761847, "epoch": 0.9243458475540387, "frac_reward_zero_std": 0.0, "grad_norm": 0.013680691830813885, "kl": 0.012185579456854612, "learning_rate": 6.860068259385666e-07, "loss": -0.0292, "num_tokens": 171229407.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.3778693675994873, "sampling/importance_sampling_ratio/mean": 0.2132938802242279, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.8026251792907715, "sampling/sampling_logp_difference/mean": 0.01343672163784504, "step": 1625, "step_time": 74.06257399986498 }, { "clip_ratio/high_max": 0.012427461566403508, "clip_ratio/high_mean": 0.003914309330866672, "clip_ratio/low_mean": 0.0013916015668655746, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005305910657625645, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.0, "completions/mean_length": 760.125, "completions/mean_terminated_length": 565.0869750976562, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.11128718871623278, "epoch": 0.9249146757679181, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.04297839477658272, "kl": 0.014482785132713616, "learning_rate": 6.80887372013652e-07, "loss": 0.1094, "num_tokens": 171331124.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.7782843112945557, "sampling/importance_sampling_ratio/mean": 0.38914602994918823, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 7.424320220947266, "sampling/sampling_logp_difference/mean": 0.01388480979949236, "step": 1626, "step_time": 74.22220340021886 }, { "clip_ratio/high_max": 0.007853038143366575, "clip_ratio/high_mean": 0.001991483091842383, "clip_ratio/low_mean": 0.0018310547020519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038225377502385527, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 944.1500244140625, "completions/mean_terminated_length": 757.8333740234375, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.11380758788436651, "epoch": 0.9254835039817975, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.017860617488622665, "kl": 0.011384559911675751, "learning_rate": 6.757679180887373e-07, "loss": 0.0461, "num_tokens": 171445810.0, "reward": 0.574999988079071, "reward_std": 0.9026314616203308, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.9544678926467896, "sampling/importance_sampling_ratio/mean": 0.3902136981487274, "sampling/importance_sampling_ratio/min": 0.003422910114750266, "sampling/sampling_logp_difference/max": 2.636375665664673, "sampling/sampling_logp_difference/mean": 0.013173920102417469, "step": 1627, "step_time": 80.70127886743285 }, { "clip_ratio/high_max": 0.007791840122081339, "clip_ratio/high_mean": 0.0019632263283710927, "clip_ratio/low_mean": 0.0017583329463377595, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037215592456050217, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 819.0, "completions/mean_length": 889.7250366210938, "completions/mean_terminated_length": 535.727294921875, "completions/min_length": 431.0, "completions/min_terminated_length": 431.0, "entropy": 0.10735285561531782, "epoch": 0.926052332195677, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01429925300180912, "kl": 0.00989018805557862, "learning_rate": 6.706484641638224e-07, "loss": 0.0709, "num_tokens": 171545207.0, "reward": 0.42500001192092896, "reward_std": 0.747217059135437, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.15000000596046448, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 2.3429172039031982, "sampling/importance_sampling_ratio/mean": 0.5325583219528198, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.335416078567505, "sampling/sampling_logp_difference/mean": 0.012320651672780514, "step": 1628, "step_time": 72.48293270426802 }, { "clip_ratio/high_max": 0.015392533387057483, "clip_ratio/high_mean": 0.006823877396527678, "clip_ratio/low_mean": 0.0021961300808470696, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009020007506478578, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 829.9249877929688, "completions/mean_terminated_length": 686.478271484375, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.15915429219603539, "epoch": 0.9266211604095563, "frac_reward_zero_std": 0.0, "grad_norm": 0.028308719396591187, "kl": 0.01520650857128203, "learning_rate": 6.655290102389078e-07, "loss": -0.1501, "num_tokens": 171655100.0, "reward": 1.0500000715255737, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.064713478088379, "sampling/importance_sampling_ratio/mean": 0.4034940302371979, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9506449699401855, "sampling/sampling_logp_difference/mean": 0.01744876056909561, "step": 1629, "step_time": 74.89061158429831 }, { "clip_ratio/high_max": 0.018182991654612124, "clip_ratio/high_mean": 0.006679803918814287, "clip_ratio/low_mean": 0.00029296875618456397, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006972772593144327, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 660.9500122070312, "completions/mean_terminated_length": 523.2413940429688, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.09288912825286388, "epoch": 0.9271899886234357, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.008016970008611679, "kl": 0.015458948910236359, "learning_rate": 6.604095563139931e-07, "loss": -0.0067, "num_tokens": 171749026.0, "reward": 1.4500000476837158, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.7250000238418579, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.609337091445923, "sampling/importance_sampling_ratio/mean": 0.38397738337516785, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3156301975250244, "sampling/sampling_logp_difference/mean": 0.012415861710906029, "step": 1630, "step_time": 70.06360134575516 }, { "clip_ratio/high_max": 0.013816303049679846, "clip_ratio/high_mean": 0.004041457636049017, "clip_ratio/low_mean": 0.001489257840148639, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005530715367058292, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 716.9000244140625, "completions/mean_terminated_length": 600.413818359375, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.1478549400344491, "epoch": 0.9277588168373151, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.0440656840801239, "kl": 0.01583856949582696, "learning_rate": 6.552901023890784e-07, "loss": -0.0169, "num_tokens": 171851630.0, "reward": 1.4500000476837158, "reward_std": 0.9044051766395569, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.7250000238418579, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.302238941192627, "sampling/importance_sampling_ratio/mean": 0.45880594849586487, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8145320415496826, "sampling/sampling_logp_difference/mean": 0.01839788444340229, "step": 1631, "step_time": 73.5936807252001 }, { "clip_ratio/high_max": 0.012996028875932097, "clip_ratio/high_mean": 0.003663996991235763, "clip_ratio/low_mean": 0.0017822265799622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00544622348388657, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.0, "completions/mean_length": 817.7250366210938, "completions/mean_terminated_length": 648.95458984375, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.08865084918215871, "epoch": 0.9283276450511946, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02630150131881237, "kl": 0.019827923621051013, "learning_rate": 6.501706484641639e-07, "loss": 0.0289, "num_tokens": 171955611.0, "reward": 0.9750000238418579, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.914665460586548, "sampling/importance_sampling_ratio/mean": 0.6158394813537598, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.129992723464966, "sampling/sampling_logp_difference/mean": 0.011466601863503456, "step": 1632, "step_time": 75.61760305985808 }, { "clip_ratio/high_max": 0.008234515436924994, "clip_ratio/high_mean": 0.003296058072010055, "clip_ratio/low_mean": 0.002075195348879788, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005371253384510055, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 830.125, "completions/mean_terminated_length": 539.3125, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.11563531216233969, "epoch": 0.928896473265074, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.025524813681840897, "kl": 0.013081400888040662, "learning_rate": 6.450511945392492e-07, "loss": -0.0431, "num_tokens": 172064848.0, "reward": 0.699999988079071, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.0229480266571045, "sampling/importance_sampling_ratio/mean": 0.3672482967376709, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1947240829467773, "sampling/sampling_logp_difference/mean": 0.013821149244904518, "step": 1633, "step_time": 74.25028956006281 }, { "clip_ratio/high_max": 0.012392377248033881, "clip_ratio/high_mean": 0.003014040456037037, "clip_ratio/low_mean": 0.003114679318969138, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006128719775006175, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 934.8250122070312, "completions/mean_terminated_length": 749.6154174804688, "completions/min_length": 553.0, "completions/min_terminated_length": 553.0, "entropy": 0.12082849815487862, "epoch": 0.9294653014789533, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021436523646116257, "kl": 0.009786929411347955, "learning_rate": 6.399317406143345e-07, "loss": 0.0602, "num_tokens": 172176881.0, "reward": 0.550000011920929, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.0320539474487305, "sampling/importance_sampling_ratio/mean": 0.48692330718040466, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4726154804229736, "sampling/sampling_logp_difference/mean": 0.013459905050694942, "step": 1634, "step_time": 76.18037123116665 }, { "clip_ratio/high_max": 0.011307388369459659, "clip_ratio/high_mean": 0.003693878563353792, "clip_ratio/low_mean": 0.0026855468895519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006379425525665283, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 842.0, "completions/mean_length": 867.375, "completions/mean_terminated_length": 632.4375, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.0757298138923943, "epoch": 0.9300341296928327, "frac_reward_zero_std": 0.0, "grad_norm": 0.04752345383167267, "kl": 0.01204636151669547, "learning_rate": 6.348122866894198e-07, "loss": -0.1462, "num_tokens": 172281792.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.4848766326904297, "sampling/importance_sampling_ratio/mean": 0.4616256654262543, "sampling/importance_sampling_ratio/min": 0.0014969839248806238, "sampling/sampling_logp_difference/max": 2.5730557441711426, "sampling/sampling_logp_difference/mean": 0.010548370890319347, "step": 1635, "step_time": 104.17259978991933 }, { "clip_ratio/high_max": 0.017320287064649165, "clip_ratio/high_mean": 0.005626627476885915, "clip_ratio/low_mean": 0.0031738281104480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008800455543678254, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 883.2000122070312, "completions/mean_terminated_length": 692.7058715820312, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.15314223803579807, "epoch": 0.9306029579067122, "frac_reward_zero_std": 0.0, "grad_norm": 0.024234093725681305, "kl": 0.017845544847659767, "learning_rate": 6.296928327645052e-07, "loss": 0.0028, "num_tokens": 172390080.0, "reward": 0.7750000357627869, "reward_std": 0.9469925761222839, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.501685857772827, "sampling/importance_sampling_ratio/mean": 0.3133489489555359, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2198262214660645, "sampling/sampling_logp_difference/mean": 0.01663070172071457, "step": 1636, "step_time": 73.84887302503921 }, { "clip_ratio/high_max": 0.00844890066946391, "clip_ratio/high_mean": 0.002484849468601169, "clip_ratio/low_mean": 0.0019042968924622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004389146313769743, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 804.625, "completions/mean_terminated_length": 562.1578979492188, "completions/min_length": 371.0, "completions/min_terminated_length": 371.0, "entropy": 0.0712524177506566, "epoch": 0.9311717861205916, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02786673791706562, "kl": 0.013907407119404525, "learning_rate": 6.245733788395905e-07, "loss": 0.1731, "num_tokens": 172495169.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.367213249206543, "sampling/importance_sampling_ratio/mean": 0.5066161155700684, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.226569175720215, "sampling/sampling_logp_difference/mean": 0.010952151380479336, "step": 1637, "step_time": 77.11696591647342 }, { "clip_ratio/high_max": 0.01658153859898448, "clip_ratio/high_mean": 0.005297913434333168, "clip_ratio/low_mean": 0.003939684964279877, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009237598394975066, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 939.0, "completions/mean_length": 842.0, "completions/mean_terminated_length": 595.7647094726562, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.14326855912804604, "epoch": 0.931740614334471, "frac_reward_zero_std": 0.0, "grad_norm": 0.03433225676417351, "kl": 0.013392278924584389, "learning_rate": 6.194539249146758e-07, "loss": -0.0552, "num_tokens": 172597465.0, "reward": 0.7750000357627869, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.9914724826812744, "sampling/importance_sampling_ratio/mean": 0.4506285786628723, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.199159860610962, "sampling/sampling_logp_difference/mean": 0.016571521759033203, "step": 1638, "step_time": 73.33153663855046 }, { "clip_ratio/high_max": 0.014455220778472722, "clip_ratio/high_mean": 0.004223353054840118, "clip_ratio/low_mean": 0.001635742193684564, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005859095312189311, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 936.0, "completions/mean_length": 807.875, "completions/mean_terminated_length": 648.1304321289062, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "entropy": 0.10537340678274632, "epoch": 0.9323094425483504, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0365561880171299, "kl": 0.016789341811090708, "learning_rate": 6.143344709897612e-07, "loss": 0.0975, "num_tokens": 172697044.0, "reward": 0.875, "reward_std": 0.8529736995697021, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.1938958168029785, "sampling/importance_sampling_ratio/mean": 0.47585469484329224, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3394479751586914, "sampling/sampling_logp_difference/mean": 0.013144470751285553, "step": 1639, "step_time": 72.17154139582999 }, { "clip_ratio/high_max": 0.018242451013065875, "clip_ratio/high_mean": 0.006749688880518079, "clip_ratio/low_mean": 0.0014335667510749772, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00818325561704114, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 855.6500244140625, "completions/mean_terminated_length": 649.888916015625, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "entropy": 0.13663740828633308, "epoch": 0.9328782707622298, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.025959255173802376, "kl": 0.011790442164056003, "learning_rate": 6.092150170648465e-07, "loss": 0.0555, "num_tokens": 172804318.0, "reward": 0.875, "reward_std": 0.9919548034667969, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.5287981033325195, "sampling/importance_sampling_ratio/mean": 0.5005897283554077, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.546626567840576, "sampling/sampling_logp_difference/mean": 0.015414359979331493, "step": 1640, "step_time": 74.00179182528518 }, { "clip_ratio/high_max": 0.0149254878051579, "clip_ratio/high_mean": 0.003938677167752758, "clip_ratio/low_mean": 0.002465820281940978, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006404497602488846, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 918.2000122070312, "completions/mean_terminated_length": 721.7142944335938, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "entropy": 0.13712007272988558, "epoch": 0.9334470989761092, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.033242929726839066, "kl": 0.014090132899582386, "learning_rate": 6.040955631399318e-07, "loss": -0.1032, "num_tokens": 172916238.0, "reward": 0.699999988079071, "reward_std": 0.9660917520523071, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.9108757972717285, "sampling/importance_sampling_ratio/mean": 0.5031899213790894, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0800907611846924, "sampling/sampling_logp_difference/mean": 0.01508784294128418, "step": 1641, "step_time": 76.362705322681 }, { "clip_ratio/high_max": 0.01582728058565408, "clip_ratio/high_mean": 0.006288926117122173, "clip_ratio/low_mean": 0.0026855468895519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008974473050329834, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 924.0, "completions/mean_length": 838.2000122070312, "completions/mean_terminated_length": 611.1111450195312, "completions/min_length": 421.0, "completions/min_terminated_length": 421.0, "entropy": 0.12471260316669941, "epoch": 0.9340159271899886, "frac_reward_zero_std": 0.0, "grad_norm": 0.04290294647216797, "kl": 0.015135027002543211, "learning_rate": 5.989761092150171e-07, "loss": 0.0405, "num_tokens": 173027606.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.799656629562378, "sampling/importance_sampling_ratio/mean": 0.416830837726593, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.516185760498047, "sampling/sampling_logp_difference/mean": 0.015734052285552025, "step": 1642, "step_time": 75.59196211979724 }, { "clip_ratio/high_max": 0.012863657728303224, "clip_ratio/high_mean": 0.0029627025069203228, "clip_ratio/low_mean": 0.0013916015886934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004354304022854194, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 924.0, "completions/mean_length": 823.1500244140625, "completions/mean_terminated_length": 551.4117431640625, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.06921183643862605, "epoch": 0.934584755403868, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024073850363492966, "kl": 0.014258243842050433, "learning_rate": 5.938566552901025e-07, "loss": -0.0322, "num_tokens": 173134012.0, "reward": 0.8500000238418579, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.087235927581787, "sampling/importance_sampling_ratio/mean": 0.2474556416273117, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1147048473358154, "sampling/sampling_logp_difference/mean": 0.012053852900862694, "step": 1643, "step_time": 74.29259867593646 }, { "clip_ratio/high_max": 0.008615813043434173, "clip_ratio/high_mean": 0.00177481553691905, "clip_ratio/low_mean": 0.0006835937620053301, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002458409217069857, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 548.0, "completions/mean_length": 780.75, "completions/mean_terminated_length": 415.875, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.11279008071869612, "epoch": 0.9351535836177475, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.007622772362083197, "kl": 0.013095035217702389, "learning_rate": 5.887372013651878e-07, "loss": 0.0212, "num_tokens": 173230890.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.2945661544799805, "sampling/importance_sampling_ratio/mean": 0.5477330088615417, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2618417739868164, "sampling/sampling_logp_difference/mean": 0.014241235330700874, "step": 1644, "step_time": 70.527639353415 }, { "clip_ratio/high_max": 0.011552542797289789, "clip_ratio/high_mean": 0.0025872382248053327, "clip_ratio/low_mean": 0.000770355123677291, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033575933484826237, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.0, "completions/mean_length": 800.5, "completions/mean_terminated_length": 617.6363525390625, "completions/min_length": 394.0, "completions/min_terminated_length": 394.0, "entropy": 0.06656140740960836, "epoch": 0.9357224118316269, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.04257246106863022, "kl": 0.00960062479134649, "learning_rate": 5.83617747440273e-07, "loss": 0.1247, "num_tokens": 173335662.0, "reward": 1.0, "reward_std": 0.96076899766922, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.7335832118988037, "sampling/importance_sampling_ratio/mean": 0.5506267547607422, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.071819305419922, "sampling/sampling_logp_difference/mean": 0.011604161001741886, "step": 1645, "step_time": 74.09462101990357 }, { "clip_ratio/high_max": 0.009755012462846935, "clip_ratio/high_mean": 0.002836826810380444, "clip_ratio/low_mean": 0.0022552550290129147, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005092081613838673, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 840.1500244140625, "completions/mean_terminated_length": 689.727294921875, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.13566498458385468, "epoch": 0.9362912400455062, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022784557193517685, "kl": 0.011514740181155503, "learning_rate": 5.784982935153584e-07, "loss": -0.0125, "num_tokens": 173437196.0, "reward": 1.024999976158142, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.290320873260498, "sampling/importance_sampling_ratio/mean": 0.46226954460144043, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.555088996887207, "sampling/sampling_logp_difference/mean": 0.01500634290277958, "step": 1646, "step_time": 73.43480523698963 }, { "clip_ratio/high_max": 0.012393418292049319, "clip_ratio/high_mean": 0.0042503024596953765, "clip_ratio/low_mean": 0.000805664065410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005055966466898099, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 826.0, "completions/mean_length": 809.1749877929688, "completions/mean_terminated_length": 451.13336181640625, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.07568139024078846, "epoch": 0.9368600682593856, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.027999162673950195, "kl": 0.014763448853045702, "learning_rate": 5.733788395904437e-07, "loss": -0.0296, "num_tokens": 173533827.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.2050962448120117, "sampling/importance_sampling_ratio/mean": 0.5024358630180359, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9065327644348145, "sampling/sampling_logp_difference/mean": 0.01084459200501442, "step": 1647, "step_time": 76.6266368792858 }, { "clip_ratio/high_max": 0.011078379233367741, "clip_ratio/high_mean": 0.003922341973520815, "clip_ratio/low_mean": 0.0011718750283762347, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005094216816360131, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 889.2750244140625, "completions/mean_terminated_length": 639.0714721679688, "completions/min_length": 444.0, "completions/min_terminated_length": 444.0, "entropy": 0.07626603730022907, "epoch": 0.9374288964732651, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01197805069386959, "kl": 0.01127865700982511, "learning_rate": 5.68259385665529e-07, "loss": 0.0398, "num_tokens": 173648526.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.0850255489349365, "sampling/importance_sampling_ratio/mean": 0.3397323191165924, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0866267681121826, "sampling/sampling_logp_difference/mean": 0.010624037124216557, "step": 1648, "step_time": 75.75930499704555 }, { "clip_ratio/high_max": 0.014276143978349864, "clip_ratio/high_mean": 0.003918128815712407, "clip_ratio/low_mean": 0.0009277343924622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0048458632081747055, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 814.0, "completions/mean_length": 890.8500366210938, "completions/mean_terminated_length": 614.3077392578125, "completions/min_length": 385.0, "completions/min_terminated_length": 385.0, "entropy": 0.06516762217506766, "epoch": 0.9379977246871445, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018995007500052452, "kl": 0.007422487600706518, "learning_rate": 5.631399317406143e-07, "loss": -0.027, "num_tokens": 173750384.0, "reward": 0.6500000357627869, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.756510853767395, "sampling/importance_sampling_ratio/mean": 0.4147810935974121, "sampling/importance_sampling_ratio/min": 0.0032845051027834415, "sampling/sampling_logp_difference/max": 2.274507761001587, "sampling/sampling_logp_difference/mean": 0.009133960120379925, "step": 1649, "step_time": 72.2608572805766 }, { "clip_ratio/high_max": 0.007207632239442319, "clip_ratio/high_mean": 0.0015875545068411157, "clip_ratio/low_mean": 0.0019042968924622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034918513847514987, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 974.0, "completions/mean_length": 932.4249877929688, "completions/mean_terminated_length": 779.800048828125, "completions/min_length": 503.0, "completions/min_terminated_length": 503.0, "entropy": 0.08845963887870312, "epoch": 0.9385665529010239, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.031240295618772507, "kl": 0.013518252177163959, "learning_rate": 5.580204778156997e-07, "loss": -0.0517, "num_tokens": 173860137.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.2209150791168213, "sampling/importance_sampling_ratio/mean": 0.49910539388656616, "sampling/importance_sampling_ratio/min": 0.0007442280184477568, "sampling/sampling_logp_difference/max": 3.3055241107940674, "sampling/sampling_logp_difference/mean": 0.012634584680199623, "step": 1650, "step_time": 74.76274869404733 }, { "clip_ratio/high_max": 0.016334469546563923, "clip_ratio/high_mean": 0.006588870892301202, "clip_ratio/low_mean": 0.0010742187605501385, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007663089549168944, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 900.0, "completions/mean_length": 809.75, "completions/mean_terminated_length": 615.90478515625, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "entropy": 0.12232107389718294, "epoch": 0.9391353811149032, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01603510230779648, "kl": 0.011516374768689275, "learning_rate": 5.52901023890785e-07, "loss": -0.035, "num_tokens": 173966239.0, "reward": 0.949999988079071, "reward_std": 0.9594336152076721, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.7013580799102783, "sampling/importance_sampling_ratio/mean": 0.4535878300666809, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2235310077667236, "sampling/sampling_logp_difference/mean": 0.014845132827758789, "step": 1651, "step_time": 72.25587563286535 }, { "clip_ratio/high_max": 0.017365060281008482, "clip_ratio/high_mean": 0.005571162007981911, "clip_ratio/low_mean": 0.0012939453263243195, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006865107279736549, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 754.5250244140625, "completions/mean_terminated_length": 609.423095703125, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.08599021006375551, "epoch": 0.9397042093287827, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03475358337163925, "kl": 0.012494880182202905, "learning_rate": 5.477815699658703e-07, "loss": -0.1839, "num_tokens": 174068116.0, "reward": 1.3000000715255737, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 1.9324687719345093, "sampling/importance_sampling_ratio/mean": 0.5170149803161621, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2718348503112793, "sampling/sampling_logp_difference/mean": 0.012668472714722157, "step": 1652, "step_time": 70.80036131921224 }, { "clip_ratio/high_max": 0.006806603283621371, "clip_ratio/high_mean": 0.0013613206538138911, "clip_ratio/low_mean": 0.000976562509094947, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002337883212021552, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 975.0, "completions/mean_length": 876.4000244140625, "completions/mean_terminated_length": 602.2857666015625, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.10587470512837172, "epoch": 0.9402730375426621, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.027813034132122993, "kl": 0.008819869195576757, "learning_rate": 5.426621160409556e-07, "loss": -0.0959, "num_tokens": 174174916.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.670672655105591, "sampling/importance_sampling_ratio/mean": 0.45460471510887146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.574504852294922, "sampling/sampling_logp_difference/mean": 0.012676936574280262, "step": 1653, "step_time": 72.85228284611367 }, { "clip_ratio/high_max": 0.01275078032631427, "clip_ratio/high_mean": 0.0038223169976845384, "clip_ratio/low_mean": 0.002050781258731149, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005873098212759942, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 832.0, "completions/mean_length": 866.4000244140625, "completions/mean_terminated_length": 498.66668701171875, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.0786933726631105, "epoch": 0.9408418657565415, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03264351189136505, "kl": 0.015139323309995234, "learning_rate": 5.37542662116041e-07, "loss": -0.0449, "num_tokens": 174289668.0, "reward": 0.6000000238418579, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.7997368574142456, "sampling/importance_sampling_ratio/mean": 0.38324326276779175, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6540586948394775, "sampling/sampling_logp_difference/mean": 0.01157744973897934, "step": 1654, "step_time": 75.62534362194128 }, { "clip_ratio/high_max": 0.014328542281873524, "clip_ratio/high_mean": 0.004169293446466327, "clip_ratio/low_mean": 0.0018859473129850812, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006055240810383111, "completions/clipped_ratio": 0.125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 716.9500122070312, "completions/mean_terminated_length": 673.085693359375, "completions/min_length": 417.0, "completions/min_terminated_length": 417.0, "entropy": 0.09164508897811174, "epoch": 0.941410693970421, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.026213131844997406, "kl": 0.015510282304603606, "learning_rate": 5.324232081911263e-07, "loss": -0.0209, "num_tokens": 174373626.0, "reward": 1.4500000476837158, "reward_std": 0.7143223881721497, "rewards/format_reward/mean": 0.875, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.8151862621307373, "sampling/importance_sampling_ratio/mean": 0.4456639885902405, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4621007442474365, "sampling/sampling_logp_difference/mean": 0.013460633344948292, "step": 1655, "step_time": 68.28614835860208 }, { "clip_ratio/high_max": 0.011254819575697184, "clip_ratio/high_mean": 0.004499732342083007, "clip_ratio/low_mean": 0.0019531249981810106, "clip_ratio/low_min": 0.00048828125, "clip_ratio/region_mean": 0.006452857400290668, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 847.4750366210938, "completions/mean_terminated_length": 670.9500122070312, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.09603064972907305, "epoch": 0.9419795221843004, "frac_reward_zero_std": 0.0, "grad_norm": 0.04416881129145622, "kl": 0.008740273537114263, "learning_rate": 5.273037542662116e-07, "loss": 0.1456, "num_tokens": 174473589.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.7654237747192383, "sampling/importance_sampling_ratio/mean": 0.503294825553894, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.332031726837158, "sampling/sampling_logp_difference/mean": 0.01210467703640461, "step": 1656, "step_time": 77.26012635813095 }, { "clip_ratio/high_max": 0.012907959520816803, "clip_ratio/high_mean": 0.004355894750915468, "clip_ratio/low_mean": 0.001977539068320766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006333433906547725, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 788.75, "completions/mean_terminated_length": 647.5999755859375, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.10466561373323202, "epoch": 0.9425483503981797, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02531317248940468, "kl": 0.013149436214007437, "learning_rate": 5.22184300341297e-07, "loss": -0.014, "num_tokens": 174581235.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 1.8612053394317627, "sampling/importance_sampling_ratio/mean": 0.316933810710907, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0545945167541504, "sampling/sampling_logp_difference/mean": 0.013618789613246918, "step": 1657, "step_time": 74.90448717493564 }, { "clip_ratio/high_max": 0.01579827710520476, "clip_ratio/high_mean": 0.005524051783140749, "clip_ratio/low_mean": 0.002061525607132353, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007585577492136508, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.0, "completions/mean_length": 852.5499877929688, "completions/mean_terminated_length": 643.0, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.12866018246859312, "epoch": 0.9431171786120591, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.016908716410398483, "kl": 0.011665514903143048, "learning_rate": 5.170648464163823e-07, "loss": 0.0053, "num_tokens": 174676801.0, "reward": 0.75, "reward_std": 0.8987170457839966, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.7981096506118774, "sampling/importance_sampling_ratio/mean": 0.2986525595188141, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0016608238220215, "sampling/sampling_logp_difference/mean": 0.014819472096860409, "step": 1658, "step_time": 70.89214384439401 }, { "clip_ratio/high_max": 0.011013429786544293, "clip_ratio/high_mean": 0.0027157324802828953, "clip_ratio/low_mean": 0.0014160156424622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004131748079089448, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 852.0, "completions/mean_length": 857.6749877929688, "completions/mean_terminated_length": 608.1875, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "entropy": 0.07125008665025234, "epoch": 0.9436860068259386, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022892579436302185, "kl": 0.01069726306013763, "learning_rate": 5.119453924914676e-07, "loss": 0.0681, "num_tokens": 174783228.0, "reward": 0.625, "reward_std": 0.8066232204437256, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.201841354370117, "sampling/importance_sampling_ratio/mean": 0.385518878698349, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.498425006866455, "sampling/sampling_logp_difference/mean": 0.011271391995251179, "step": 1659, "step_time": 73.2169572650455 }, { "clip_ratio/high_max": 0.015847578062675893, "clip_ratio/high_mean": 0.004200831943308003, "clip_ratio/low_mean": 0.002172851556679234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006373683572746813, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 847.0, "completions/mean_length": 867.5750122070312, "completions/mean_terminated_length": 502.5833435058594, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.10816481616348028, "epoch": 0.944254835039818, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03263373300433159, "kl": 0.016494082869030535, "learning_rate": 5.068259385665529e-07, "loss": -0.0133, "num_tokens": 174889323.0, "reward": 0.550000011920929, "reward_std": 0.8755950927734375, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.9092371463775635, "sampling/importance_sampling_ratio/mean": 0.3865593373775482, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.595099925994873, "sampling/sampling_logp_difference/mean": 0.013142821379005909, "step": 1660, "step_time": 70.79883177182637 }, { "clip_ratio/high_max": 0.0174201603513211, "clip_ratio/high_mean": 0.006987948174355552, "clip_ratio/low_mean": 0.0013427734375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008330721640959382, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 960.0, "completions/mean_length": 694.8500366210938, "completions/mean_terminated_length": 451.5652160644531, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.09893292095512152, "epoch": 0.9448236632536974, "frac_reward_zero_std": 0.0, "grad_norm": 0.015858100727200508, "kl": 0.015307635243516415, "learning_rate": 5.017064846416383e-07, "loss": -0.0382, "num_tokens": 174990541.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.889569044113159, "sampling/importance_sampling_ratio/mean": 0.34113192558288574, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.614353895187378, "sampling/sampling_logp_difference/mean": 0.013533929362893105, "step": 1661, "step_time": 72.0191572678741 }, { "clip_ratio/high_max": 0.00964425946585834, "clip_ratio/high_mean": 0.0025829082005657256, "clip_ratio/low_mean": 0.0013183593746362021, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390126759884879, "completions/clipped_ratio": 0.7750000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 957.4249877929688, "completions/mean_terminated_length": 728.1111450195312, "completions/min_length": 418.0, "completions/min_terminated_length": 418.0, "entropy": 0.08908907882869244, "epoch": 0.9453924914675768, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02180372178554535, "kl": 0.00810040399665013, "learning_rate": 4.965870307167236e-07, "loss": 0.0521, "num_tokens": 175101198.0, "reward": 0.32500001788139343, "reward_std": 0.655841588973999, "rewards/format_reward/mean": 0.22499999403953552, "rewards/format_reward/std": 0.42290204763412476, "rewards/gpt_judge_reward/mean": 0.10000000149011612, "rewards/gpt_judge_reward/std": 0.3038218021392822, "sampling/importance_sampling_ratio/max": 2.7769036293029785, "sampling/importance_sampling_ratio/mean": 0.4046157896518707, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.92551589012146, "sampling/sampling_logp_difference/mean": 0.010601460002362728, "step": 1662, "step_time": 73.76635852106847 }, { "clip_ratio/high_max": 0.010123294952791184, "clip_ratio/high_mean": 0.003622249496402219, "clip_ratio/low_mean": 0.0010498046976863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004672054230468348, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 931.625, "completions/mean_terminated_length": 739.769287109375, "completions/min_length": 533.0, "completions/min_terminated_length": 533.0, "entropy": 0.10773729998618364, "epoch": 0.9459613196814562, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.009017243050038815, "kl": 0.010864840529393405, "learning_rate": 4.914675767918089e-07, "loss": -0.0039, "num_tokens": 175203719.0, "reward": 0.5250000357627869, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.169711112976074, "sampling/importance_sampling_ratio/mean": 0.3099155128002167, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.316736936569214, "sampling/sampling_logp_difference/mean": 0.012530433014035225, "step": 1663, "step_time": 74.5046070269309 }, { "clip_ratio/high_max": 0.004027200513519347, "clip_ratio/high_mean": 0.0008054400968831033, "clip_ratio/low_mean": 0.0006103515770519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014157916739350185, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 846.4000244140625, "completions/mean_terminated_length": 629.3333129882812, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.07901090569794178, "epoch": 0.9465301478953356, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.01655430905520916, "kl": 0.008352818142157048, "learning_rate": 4.863481228668943e-07, "loss": 0.0315, "num_tokens": 175318575.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.7124452590942383, "sampling/importance_sampling_ratio/mean": 0.45813244581222534, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.118856430053711, "sampling/sampling_logp_difference/mean": 0.01146224606782198, "step": 1664, "step_time": 75.14643327868544 }, { "clip_ratio/high_max": 0.013033035211265087, "clip_ratio/high_mean": 0.004884747817413881, "clip_ratio/low_mean": 0.002249747616588138, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007134495419450104, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 791.75, "completions/mean_terminated_length": 581.6190795898438, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.07454544166103005, "epoch": 0.947098976109215, "frac_reward_zero_std": 0.0, "grad_norm": 0.043005023151636124, "kl": 0.01607033284381032, "learning_rate": 4.812286689419796e-07, "loss": -0.0078, "num_tokens": 175424429.0, "reward": 0.800000011920929, "reward_std": 0.9114654064178467, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.291952610015869, "sampling/importance_sampling_ratio/mean": 0.4595300257205963, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.050690650939941, "sampling/sampling_logp_difference/mean": 0.010466625913977623, "step": 1665, "step_time": 76.16562358802184 }, { "clip_ratio/high_max": 0.008072963799349964, "clip_ratio/high_mean": 0.0021401681879069656, "clip_ratio/low_mean": 0.0015606790402671322, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003700847242726013, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 748.0750122070312, "completions/mean_terminated_length": 564.125, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.11828751768916845, "epoch": 0.9476678043230944, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.008776889182627201, "kl": 0.01483392424415797, "learning_rate": 4.7610921501706486e-07, "loss": -0.0101, "num_tokens": 175525760.0, "reward": 0.949999988079071, "reward_std": 0.8755950331687927, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.4997611045837402, "sampling/importance_sampling_ratio/mean": 0.4288899898529053, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6497983932495117, "sampling/sampling_logp_difference/mean": 0.015902694314718246, "step": 1666, "step_time": 69.9336967074778 }, { "clip_ratio/high_max": 0.006879345804918557, "clip_ratio/high_mean": 0.0015060775185702369, "clip_ratio/low_mean": 0.0006347656380967237, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021408431057352573, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 920.0, "completions/mean_length": 866.5250244140625, "completions/mean_terminated_length": 604.0667114257812, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "entropy": 0.10304732900112867, "epoch": 0.9482366325369739, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.009384448640048504, "kl": 0.010865448391996324, "learning_rate": 4.7098976109215013e-07, "loss": 0.0414, "num_tokens": 175639885.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.405578136444092, "sampling/importance_sampling_ratio/mean": 0.3741755783557892, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6046788692474365, "sampling/sampling_logp_difference/mean": 0.012611444108188152, "step": 1667, "step_time": 75.60662186611444 }, { "clip_ratio/high_max": 0.010479351098183542, "clip_ratio/high_mean": 0.0024274352472275496, "clip_ratio/low_mean": 0.0005615234440483619, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029889586512581445, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 800.0, "completions/mean_length": 809.4500122070312, "completions/mean_terminated_length": 451.86669921875, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.0871417410671711, "epoch": 0.9488054607508533, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.010814027860760689, "kl": 0.013403323420789093, "learning_rate": 4.6587030716723546e-07, "loss": -0.0181, "num_tokens": 175748791.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 1.3347067832946777, "sampling/importance_sampling_ratio/mean": 0.28621211647987366, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3140954971313477, "sampling/sampling_logp_difference/mean": 0.012339639477431774, "step": 1668, "step_time": 72.05189044866711 }, { "clip_ratio/high_max": 0.015937421354465187, "clip_ratio/high_mean": 0.005812563613289967, "clip_ratio/low_mean": 0.0011474609636934474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006960024533327669, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 908.0, "completions/mean_length": 755.1500244140625, "completions/mean_terminated_length": 625.7037353515625, "completions/min_length": 396.0, "completions/min_terminated_length": 396.0, "entropy": 0.11093231290578842, "epoch": 0.9493742889647326, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018143653869628906, "kl": 0.012382896733470261, "learning_rate": 4.607508532423208e-07, "loss": -0.0222, "num_tokens": 175857685.0, "reward": 1.350000023841858, "reward_std": 0.9486832618713379, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.675000011920929, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 1.9763003587722778, "sampling/importance_sampling_ratio/mean": 0.2547950744628906, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.75275719165802, "sampling/sampling_logp_difference/mean": 0.014637673273682594, "step": 1669, "step_time": 73.9855700719636 }, { "clip_ratio/high_max": 0.015794786741025746, "clip_ratio/high_mean": 0.006488504383014515, "clip_ratio/low_mean": 0.0017337584013148444, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008222262724302709, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 712.2250366210938, "completions/mean_terminated_length": 621.7096557617188, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.13486155960708857, "epoch": 0.949943117178612, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.023520098999142647, "kl": 0.015716913621872663, "learning_rate": 4.5563139931740617e-07, "loss": 0.052, "num_tokens": 175960366.0, "reward": 1.25, "reward_std": 0.8086075186729431, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.163201093673706, "sampling/importance_sampling_ratio/mean": 0.3496212363243103, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6075005531311035, "sampling/sampling_logp_difference/mean": 0.017293015494942665, "step": 1670, "step_time": 71.65503498818725 }, { "clip_ratio/high_max": 0.012353239231742918, "clip_ratio/high_mean": 0.0036614859709516168, "clip_ratio/low_mean": 0.0013019055149925407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004963391547789797, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 688.0, "completions/mean_length": 859.2000122070312, "completions/mean_terminated_length": 553.1428833007812, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.10958964936435223, "epoch": 0.9505119453924915, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04803837463259697, "kl": 0.01251107145799324, "learning_rate": 4.505119453924915e-07, "loss": -0.0184, "num_tokens": 176064958.0, "reward": 0.4749999940395355, "reward_std": 0.715667188167572, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 2.465695381164551, "sampling/importance_sampling_ratio/mean": 0.32049956917762756, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3471193313598633, "sampling/sampling_logp_difference/mean": 0.01325790025293827, "step": 1671, "step_time": 72.14463424729183 }, { "clip_ratio/high_max": 0.014273354317992926, "clip_ratio/high_mean": 0.0036616881261579692, "clip_ratio/low_mean": 0.0028590330184670165, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006520721246488392, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 943.0, "completions/mean_length": 853.6500244140625, "completions/mean_terminated_length": 623.1764526367188, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.10139879025518894, "epoch": 0.9510807736063709, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02788863331079483, "kl": 0.01391236181370914, "learning_rate": 4.4539249146757683e-07, "loss": -0.0652, "num_tokens": 176171280.0, "reward": 0.675000011920929, "reward_std": 0.858964741230011, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.0741310119628906, "sampling/importance_sampling_ratio/mean": 0.35399338603019714, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3818602561950684, "sampling/sampling_logp_difference/mean": 0.013008142821490765, "step": 1672, "step_time": 73.1652428000234 }, { "clip_ratio/high_max": 0.008655116107547656, "clip_ratio/high_mean": 0.00237931968877092, "clip_ratio/low_mean": 0.0028076171729480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00518693684716709, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 954.8250122070312, "completions/mean_terminated_length": 747.2999877929688, "completions/min_length": 515.0, "completions/min_terminated_length": 515.0, "entropy": 0.1322889719158411, "epoch": 0.9516496018202503, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.012399374507367611, "kl": 0.01077883888501674, "learning_rate": 4.4027303754266216e-07, "loss": 0.0176, "num_tokens": 176285817.0, "reward": 0.45000001788139343, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.5209336280822754, "sampling/importance_sampling_ratio/mean": 0.3000192642211914, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0847008228302, "sampling/sampling_logp_difference/mean": 0.014335083775222301, "step": 1673, "step_time": 77.09685609512962 }, { "clip_ratio/high_max": 0.014259512885473669, "clip_ratio/high_mean": 0.004849414981435984, "clip_ratio/low_mean": 0.002270507815410383, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.007119922811398283, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 863.4249877929688, "completions/mean_terminated_length": 622.5625, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.10615277383476496, "epoch": 0.9522184300341296, "frac_reward_zero_std": 0.0, "grad_norm": 0.03363976627588272, "kl": 0.01765421056188643, "learning_rate": 4.351535836177475e-07, "loss": -0.0183, "num_tokens": 176390858.0, "reward": 0.7250000238418579, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.030104875564575, "sampling/importance_sampling_ratio/mean": 0.3223192095756531, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.889410972595215, "sampling/sampling_logp_difference/mean": 0.013092401437461376, "step": 1674, "step_time": 74.318826953182 }, { "clip_ratio/high_max": 0.004267308802809566, "clip_ratio/high_mean": 0.0010295181236870121, "clip_ratio/low_mean": 0.0018066406046273187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002836158746504225, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 845.625, "completions/mean_terminated_length": 475.15386962890625, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.060971316415816545, "epoch": 0.9527872582480091, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021983835846185684, "kl": 0.007602727098856121, "learning_rate": 4.300341296928328e-07, "loss": 0.0497, "num_tokens": 176497435.0, "reward": 0.625, "reward_std": 0.9250779747962952, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 1.7652543783187866, "sampling/importance_sampling_ratio/mean": 0.4153764843940735, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9809703826904297, "sampling/sampling_logp_difference/mean": 0.008926590904593468, "step": 1675, "step_time": 70.22861040895805 }, { "clip_ratio/high_max": 0.017759523587301373, "clip_ratio/high_mean": 0.0057482449628878385, "clip_ratio/low_mean": 0.0023108008899725974, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008059045881964266, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 923.0, "completions/mean_length": 723.625, "completions/mean_terminated_length": 561.8846435546875, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.13922585546970367, "epoch": 0.9533560864618885, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0392015278339386, "kl": 0.014977191807702184, "learning_rate": 4.249146757679181e-07, "loss": 0.0899, "num_tokens": 176604284.0, "reward": 1.225000023841858, "reward_std": 0.9469925761222839, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.6524477005004883, "sampling/importance_sampling_ratio/mean": 0.401851087808609, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6748590469360352, "sampling/sampling_logp_difference/mean": 0.01653306558728218, "step": 1676, "step_time": 71.80707248626277 }, { "clip_ratio/high_max": 0.01431130594573915, "clip_ratio/high_mean": 0.005046254780609161, "clip_ratio/low_mean": 0.0019637313089333475, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007009986147750169, "completions/clipped_ratio": 0.20000000298023224, "completions/max_length": 1024.0, "completions/max_terminated_length": 924.0, "completions/mean_length": 637.75, "completions/mean_terminated_length": 541.1875, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.11744603328406811, "epoch": 0.9539249146757679, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.09224096685647964, "kl": 0.013125219731591642, "learning_rate": 4.197952218430034e-07, "loss": -0.2514, "num_tokens": 176705946.0, "reward": 1.4500000476837158, "reward_std": 0.8149248957633972, "rewards/format_reward/mean": 0.800000011920929, "rewards/format_reward/std": 0.40509578585624695, "rewards/gpt_judge_reward/mean": 0.6499999761581421, "rewards/gpt_judge_reward/std": 0.48304590582847595, "sampling/importance_sampling_ratio/max": 2.6079905033111572, "sampling/importance_sampling_ratio/mean": 0.6190412640571594, "sampling/importance_sampling_ratio/min": 0.0006479606381617486, "sampling/sampling_logp_difference/max": 2.342150926589966, "sampling/sampling_logp_difference/mean": 0.016468731686472893, "step": 1677, "step_time": 74.51423425390385 }, { "clip_ratio/high_max": 0.008636503363959491, "clip_ratio/high_mean": 0.0025492420536465943, "clip_ratio/low_mean": 0.0016845703285071068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004233812345773913, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 731.75, "completions/mean_terminated_length": 492.6363830566406, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.1093426663428545, "epoch": 0.9544937428896473, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.054098471999168396, "kl": 0.012451844988390803, "learning_rate": 4.1467576791808874e-07, "loss": -0.0553, "num_tokens": 176811888.0, "reward": 1.0500000715255737, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.5928828716278076, "sampling/importance_sampling_ratio/mean": 0.29627957940101624, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.772716760635376, "sampling/sampling_logp_difference/mean": 0.015111214481294155, "step": 1678, "step_time": 74.65918438928202 }, { "clip_ratio/high_max": 0.009742934023961425, "clip_ratio/high_mean": 0.0029239153082016855, "clip_ratio/low_mean": 0.002197265625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0051211809914093465, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 853.625, "completions/mean_terminated_length": 665.3157958984375, "completions/min_length": 385.0, "completions/min_terminated_length": 385.0, "entropy": 0.14009204506874084, "epoch": 0.9550625711035268, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.027512729167938232, "kl": 0.01277386408764869, "learning_rate": 4.0955631399317407e-07, "loss": -0.0766, "num_tokens": 176920089.0, "reward": 0.925000011920929, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9961578845977783, "sampling/importance_sampling_ratio/mean": 0.4201586842536926, "sampling/importance_sampling_ratio/min": 0.00047018000623211265, "sampling/sampling_logp_difference/max": 5.028399467468262, "sampling/sampling_logp_difference/mean": 0.01683853380382061, "step": 1679, "step_time": 72.91904015885666 }, { "clip_ratio/high_max": 0.014837025722954422, "clip_ratio/high_mean": 0.00562626140890643, "clip_ratio/low_mean": 0.0017578125225554686, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007384074036963284, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 829.2000122070312, "completions/mean_terminated_length": 634.4000244140625, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "entropy": 0.08213567361235619, "epoch": 0.9556313993174061, "frac_reward_zero_std": 0.0, "grad_norm": 0.01682010479271412, "kl": 0.011819146806374192, "learning_rate": 4.044368600682594e-07, "loss": -0.0123, "num_tokens": 177028225.0, "reward": 1.0, "reward_std": 1.0127394199371338, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.5949585437774658, "sampling/importance_sampling_ratio/mean": 0.2911635637283325, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6659610271453857, "sampling/sampling_logp_difference/mean": 0.011461969465017319, "step": 1680, "step_time": 72.68475718004629 }, { "clip_ratio/high_max": 0.016339582856744528, "clip_ratio/high_mean": 0.005326104641426355, "clip_ratio/low_mean": 0.0027940873405896127, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008120191749185324, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 967.0, "completions/mean_length": 712.0, "completions/mean_terminated_length": 593.6551513671875, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.15522782504558563, "epoch": 0.9562002275312855, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022332537919282913, "kl": 0.01682381972204894, "learning_rate": 3.9931740614334473e-07, "loss": 0.0286, "num_tokens": 177131521.0, "reward": 1.25, "reward_std": 0.8697184920310974, "rewards/format_reward/mean": 0.7250000238418579, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.409338116645813, "sampling/importance_sampling_ratio/mean": 0.29492244124412537, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7045364379882812, "sampling/sampling_logp_difference/mean": 0.01827256754040718, "step": 1681, "step_time": 73.72878494346514 }, { "clip_ratio/high_max": 0.0021691974252462387, "clip_ratio/high_mean": 0.00043383947922848165, "clip_ratio/low_mean": 0.0009033203223225428, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013371598015510244, "completions/clipped_ratio": 0.9750000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 922.0, "completions/mean_length": 1021.4500122070312, "completions/mean_terminated_length": 922.0, "completions/min_length": 922.0, "completions/min_terminated_length": 922.0, "entropy": 0.07179279578849673, "epoch": 0.9567690557451649, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.0067316023632884026, "kl": 0.008045301248785108, "learning_rate": 3.9419795221843006e-07, "loss": 0.0402, "num_tokens": 177236635.0, "reward": 0.05000000074505806, "reward_std": 0.3162277638912201, "rewards/format_reward/mean": 0.02500000037252903, "rewards/format_reward/std": 0.15811389684677124, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.30008602142334, "sampling/importance_sampling_ratio/mean": 0.5855060815811157, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1296722888946533, "sampling/sampling_logp_difference/mean": 0.00956412311643362, "step": 1682, "step_time": 72.80904787569307 }, { "clip_ratio/high_max": 0.016189485788345337, "clip_ratio/high_mean": 0.005434853810584173, "clip_ratio/low_mean": 0.0018310546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00726590835256502, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 827.4500122070312, "completions/mean_terminated_length": 587.2222290039062, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.09808160737156868, "epoch": 0.9573378839590444, "frac_reward_zero_std": 0.0, "grad_norm": 0.016573429107666016, "kl": 0.012139348196797073, "learning_rate": 3.890784982935154e-07, "loss": 0.0791, "num_tokens": 177343797.0, "reward": 0.9000000357627869, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.0046515464782715, "sampling/importance_sampling_ratio/mean": 0.23728111386299133, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8322267532348633, "sampling/sampling_logp_difference/mean": 0.011722604744136333, "step": 1683, "step_time": 75.55767371598631 }, { "clip_ratio/high_max": 0.011404406046494842, "clip_ratio/high_mean": 0.0026299682940589264, "clip_ratio/low_mean": 0.0014892578110448085, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004119226097827777, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 941.0, "completions/mean_length": 771.4249877929688, "completions/mean_terminated_length": 564.7727661132812, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.1306437598541379, "epoch": 0.9579067121729238, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01127570029348135, "kl": 0.013792053912766278, "learning_rate": 3.839590443686007e-07, "loss": -0.0427, "num_tokens": 177449670.0, "reward": 1.0500000715255737, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 1.4643797874450684, "sampling/importance_sampling_ratio/mean": 0.24924853444099426, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6465580463409424, "sampling/sampling_logp_difference/mean": 0.017209915444254875, "step": 1684, "step_time": 81.32039949204773 }, { "clip_ratio/high_max": 0.008129568945150822, "clip_ratio/high_mean": 0.0023331338597927243, "clip_ratio/low_mean": 0.0006103515643189894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002943485393188894, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 788.625, "completions/mean_terminated_length": 647.3999633789062, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.10571761149913073, "epoch": 0.9584755403868032, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.04060930758714676, "kl": 0.014619283843785524, "learning_rate": 3.78839590443686e-07, "loss": -0.0937, "num_tokens": 177557335.0, "reward": 1.100000023841858, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.7572100162506104, "sampling/importance_sampling_ratio/mean": 0.4794014096260071, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6617166996002197, "sampling/sampling_logp_difference/mean": 0.0142331188544631, "step": 1685, "step_time": 104.58092432911508 }, { "clip_ratio/high_max": 0.007480569824110717, "clip_ratio/high_mean": 0.0018875033420044929, "clip_ratio/low_mean": 0.0017822265508584678, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003669729849207215, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 770.1749877929688, "completions/mean_terminated_length": 516.3500366210938, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.12155437376350164, "epoch": 0.9590443686006825, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.01875731348991394, "kl": 0.015404372476041317, "learning_rate": 3.737201365187713e-07, "loss": -0.0421, "num_tokens": 177665102.0, "reward": 0.7750000357627869, "reward_std": 0.8619447350502014, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.871363878250122, "sampling/importance_sampling_ratio/mean": 0.29274335503578186, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.438222885131836, "sampling/sampling_logp_difference/mean": 0.015701869502663612, "step": 1686, "step_time": 73.4844577612821 }, { "clip_ratio/high_max": 0.01285181485582143, "clip_ratio/high_mean": 0.003862103505525738, "clip_ratio/low_mean": 0.0015869140697759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005449017480714247, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 905.0, "completions/mean_length": 856.2750244140625, "completions/mean_terminated_length": 544.7857666015625, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.08773484081029892, "epoch": 0.959613196814562, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03783660754561424, "kl": 0.01322794274892658, "learning_rate": 3.6860068259385664e-07, "loss": -0.0762, "num_tokens": 177769897.0, "reward": 0.675000011920929, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434166073799133, "sampling/importance_sampling_ratio/max": 2.133455514907837, "sampling/importance_sampling_ratio/mean": 0.4322325587272644, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.263256788253784, "sampling/sampling_logp_difference/mean": 0.010943799279630184, "step": 1687, "step_time": 71.53636159957387 }, { "clip_ratio/high_max": 0.00823188154026866, "clip_ratio/high_mean": 0.0023634516837773845, "clip_ratio/low_mean": 0.0015380859513243195, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003901537536876276, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 934.6000366210938, "completions/mean_terminated_length": 698.9091186523438, "completions/min_length": 439.0, "completions/min_terminated_length": 439.0, "entropy": 0.07017946103587747, "epoch": 0.9601820250284414, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03040291741490364, "kl": 0.0074377768323756754, "learning_rate": 3.6348122866894197e-07, "loss": 0.1047, "num_tokens": 177884233.0, "reward": 0.30000001192092896, "reward_std": 0.5163977742195129, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 2.70511794090271, "sampling/importance_sampling_ratio/mean": 0.5276604294776917, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.027223825454712, "sampling/sampling_logp_difference/mean": 0.009785431437194347, "step": 1688, "step_time": 76.54638568428345 }, { "clip_ratio/high_max": 0.009552828210871667, "clip_ratio/high_mean": 0.002612527459859848, "clip_ratio/low_mean": 0.0027832031792058842, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005395730608142912, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 816.3500366210938, "completions/mean_terminated_length": 562.5555419921875, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.11613326147198677, "epoch": 0.9607508532423208, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.042347803711891174, "kl": 0.01054849021602422, "learning_rate": 3.583617747440273e-07, "loss": 0.0274, "num_tokens": 177989759.0, "reward": 0.800000011920929, "reward_std": 0.9391759037971497, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.0835254192352295, "sampling/importance_sampling_ratio/mean": 0.423602432012558, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.439836025238037, "sampling/sampling_logp_difference/mean": 0.014222060330212116, "step": 1689, "step_time": 72.04723610612564 }, { "clip_ratio/high_max": 0.014451054972596467, "clip_ratio/high_mean": 0.0038333555858116597, "clip_ratio/low_mean": 0.0018310546729480848, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005664410186000168, "completions/clipped_ratio": 0.550000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 836.75, "completions/mean_terminated_length": 607.888916015625, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.11570221185684204, "epoch": 0.9613196814562003, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.008556737564504147, "kl": 0.012085223919712007, "learning_rate": 3.5324232081911263e-07, "loss": 0.0179, "num_tokens": 178099917.0, "reward": 0.8500000238418579, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.7375130653381348, "sampling/importance_sampling_ratio/mean": 0.2673730254173279, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.217766284942627, "sampling/sampling_logp_difference/mean": 0.013310909271240234, "step": 1690, "step_time": 73.33752498589456 }, { "clip_ratio/high_max": 0.004305612412281334, "clip_ratio/high_mean": 0.000861122491187416, "clip_ratio/low_mean": 0.0014160156424622983, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022771381482016295, "completions/clipped_ratio": 0.925000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 882.0, "completions/mean_length": 1001.625, "completions/mean_terminated_length": 725.6666870117188, "completions/min_length": 558.0, "completions/min_terminated_length": 558.0, "entropy": 0.1301664039492607, "epoch": 0.9618885096700797, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.040593378245830536, "kl": 0.010349436779506505, "learning_rate": 3.4812286689419796e-07, "loss": -0.075, "num_tokens": 178212678.0, "reward": 0.15000000596046448, "reward_std": 0.5334935784339905, "rewards/format_reward/mean": 0.07500000298023224, "rewards/format_reward/std": 0.26674678921699524, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.6077630519866943, "sampling/importance_sampling_ratio/mean": 0.3520447313785553, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9408230781555176, "sampling/sampling_logp_difference/mean": 0.013295876793563366, "step": 1691, "step_time": 75.06225563609041 }, { "clip_ratio/high_max": 0.016460616898257285, "clip_ratio/high_mean": 0.0058180271298624575, "clip_ratio/low_mean": 0.001440429718059022, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007258456840645522, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 917.0, "completions/mean_length": 801.9000244140625, "completions/mean_terminated_length": 579.7999877929688, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.12454508896917105, "epoch": 0.962457337883959, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021083328872919083, "kl": 0.015023700427263975, "learning_rate": 3.430034129692833e-07, "loss": -0.0583, "num_tokens": 178315842.0, "reward": 0.7750000357627869, "reward_std": 0.8619447350502014, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 2.400961399078369, "sampling/importance_sampling_ratio/mean": 0.37290534377098083, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.973878026008606, "sampling/sampling_logp_difference/mean": 0.014459646306931973, "step": 1692, "step_time": 75.70355705311522 }, { "clip_ratio/high_max": 0.009720822214148939, "clip_ratio/high_mean": 0.002095221832860261, "clip_ratio/low_mean": 0.0020996093517169356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004194831155473366, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 979.0, "completions/mean_length": 857.75, "completions/mean_terminated_length": 580.6666870117188, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "entropy": 0.10138675197958946, "epoch": 0.9630261660978384, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03802717849612236, "kl": 0.012913016602396965, "learning_rate": 3.3788395904436866e-07, "loss": -0.0394, "num_tokens": 178424456.0, "reward": 0.75, "reward_std": 0.9805806875228882, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.7937800884246826, "sampling/importance_sampling_ratio/mean": 0.36568117141723633, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.215329170227051, "sampling/sampling_logp_difference/mean": 0.013342796824872494, "step": 1693, "step_time": 72.12364563671872 }, { "clip_ratio/high_max": 0.010368272429332137, "clip_ratio/high_mean": 0.00230127212125808, "clip_ratio/low_mean": 0.0011172556405654177, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034185277327196673, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 768.1749877929688, "completions/mean_terminated_length": 614.6799926757812, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.10234504286199808, "epoch": 0.9635949943117179, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.027791932225227356, "kl": 0.014011504827067256, "learning_rate": 3.327645051194539e-07, "loss": -0.0458, "num_tokens": 178534319.0, "reward": 1.2000000476837158, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.4593826532363892, "sampling/importance_sampling_ratio/mean": 0.32298150658607483, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1796977519989014, "sampling/sampling_logp_difference/mean": 0.014125260524451733, "step": 1694, "step_time": 72.99311247211881 }, { "clip_ratio/high_max": 0.012005983386188745, "clip_ratio/high_mean": 0.002631258044857532, "clip_ratio/low_mean": 0.002490234372089617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005121492373291403, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 913.0, "completions/mean_length": 915.75, "completions/mean_terminated_length": 690.923095703125, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.10124098975211382, "epoch": 0.9641638225255973, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.01865367218852043, "kl": 0.010580665606539696, "learning_rate": 3.276450511945392e-07, "loss": -0.0125, "num_tokens": 178634381.0, "reward": 0.45000001788139343, "reward_std": 0.7143223881721497, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.125, "rewards/gpt_judge_reward/std": 0.3349320888519287, "sampling/importance_sampling_ratio/max": 1.8443262577056885, "sampling/importance_sampling_ratio/mean": 0.36538591980934143, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4971375465393066, "sampling/sampling_logp_difference/mean": 0.012233984656631947, "step": 1695, "step_time": 72.10389934363775 }, { "clip_ratio/high_max": 0.013498877757228911, "clip_ratio/high_mean": 0.003912700834916905, "clip_ratio/low_mean": 0.003014842193806544, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006927543203346431, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 803.0, "completions/mean_length": 765.5750122070312, "completions/mean_terminated_length": 507.1499938964844, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.11413755360990763, "epoch": 0.9647326507394767, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.0166477058082819, "kl": 0.017432041466236115, "learning_rate": 3.225255972696246e-07, "loss": -0.0121, "num_tokens": 178728396.0, "reward": 0.7250000238418579, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 1.3858062028884888, "sampling/importance_sampling_ratio/mean": 0.23110513389110565, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.167224168777466, "sampling/sampling_logp_difference/mean": 0.015000591054558754, "step": 1696, "step_time": 70.17691734456457 }, { "clip_ratio/high_max": 0.013139538583345711, "clip_ratio/high_mean": 0.004670622118283063, "clip_ratio/low_mean": 0.0010986328125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005769254930783063, "completions/clipped_ratio": 0.22500000894069672, "completions/max_length": 1024.0, "completions/max_terminated_length": 910.0, "completions/mean_length": 646.2999877929688, "completions/mean_terminated_length": 536.6451416015625, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.12588867917656898, "epoch": 0.965301478953356, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02545144222676754, "kl": 0.016901212162338197, "learning_rate": 3.174061433447099e-07, "loss": -0.0727, "num_tokens": 178831488.0, "reward": 1.5500000715255737, "reward_std": 0.8458040952682495, "rewards/format_reward/mean": 0.7749999761581421, "rewards/format_reward/std": 0.42290210723876953, "rewards/gpt_judge_reward/mean": 0.7749999761581421, "rewards/gpt_judge_reward/std": 0.42290210723876953, "sampling/importance_sampling_ratio/max": 2.6585466861724854, "sampling/importance_sampling_ratio/mean": 0.6417402029037476, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9860188961029053, "sampling/sampling_logp_difference/mean": 0.015851736068725586, "step": 1697, "step_time": 71.32863370515406 }, { "clip_ratio/high_max": 0.014472462033154443, "clip_ratio/high_mean": 0.004664807463996112, "clip_ratio/low_mean": 0.0034423827892169356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008107190136797726, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 888.6500244140625, "completions/mean_terminated_length": 663.0667114257812, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "entropy": 0.12664903700351715, "epoch": 0.9658703071672355, "frac_reward_zero_std": 0.0, "grad_norm": 0.20958511531352997, "kl": 0.09550780768040568, "learning_rate": 3.1228668941979525e-07, "loss": 0.0917, "num_tokens": 178935850.0, "reward": 0.6500000357627869, "reward_std": 0.8929926753044128, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220258831977844, "sampling/importance_sampling_ratio/max": 1.8010706901550293, "sampling/importance_sampling_ratio/mean": 0.274675190448761, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.808355331420898, "sampling/sampling_logp_difference/mean": 0.015483100898563862, "step": 1698, "step_time": 76.12897777254693 }, { "clip_ratio/high_max": 0.01170085568446666, "clip_ratio/high_mean": 0.0037239603261696175, "clip_ratio/low_mean": 0.0021240234782453626, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005847983760759234, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 904.9500122070312, "completions/mean_terminated_length": 657.6923217773438, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.11139464471489191, "epoch": 0.9664391353811149, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.017580464482307434, "kl": 0.013303772720973939, "learning_rate": 3.071672354948806e-07, "loss": 0.0672, "num_tokens": 179044624.0, "reward": 0.574999988079071, "reward_std": 0.873762845993042, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.5927536487579346, "sampling/importance_sampling_ratio/mean": 0.4004925787448883, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.105024814605713, "sampling/sampling_logp_difference/mean": 0.013154714368283749, "step": 1699, "step_time": 73.68855096609332 }, { "clip_ratio/high_max": 0.007418477674946189, "clip_ratio/high_mean": 0.0021019565319875255, "clip_ratio/low_mean": 0.0008056640745053301, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002907620626501739, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.0, "completions/mean_length": 890.4000244140625, "completions/mean_terminated_length": 642.2857666015625, "completions/min_length": 479.0, "completions/min_terminated_length": 479.0, "entropy": 0.07669293321669102, "epoch": 0.9670079635949943, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.015442450530827045, "kl": 0.01010715530719608, "learning_rate": 3.020477815699659e-07, "loss": -0.0349, "num_tokens": 179153712.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 1.5846757888793945, "sampling/importance_sampling_ratio/mean": 0.2376437485218048, "sampling/importance_sampling_ratio/min": 0.0015928905922919512, "sampling/sampling_logp_difference/max": 2.071833372116089, "sampling/sampling_logp_difference/mean": 0.010843189433217049, "step": 1700, "step_time": 72.867514121579 }, { "clip_ratio/high_max": 0.005911035230383277, "clip_ratio/high_mean": 0.0015728422731626779, "clip_ratio/low_mean": 0.0020704155213024933, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036432578053791076, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 787.8500366210938, "completions/mean_terminated_length": 613.3043823242188, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.11127272807061672, "epoch": 0.9675767918088737, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018523652106523514, "kl": 0.011287933739367872, "learning_rate": 2.9692832764505124e-07, "loss": 0.1, "num_tokens": 179258234.0, "reward": 1.0750000476837158, "reward_std": 0.9971112012863159, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057363510131836, "sampling/importance_sampling_ratio/max": 1.7911009788513184, "sampling/importance_sampling_ratio/mean": 0.45075708627700806, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2586886882781982, "sampling/sampling_logp_difference/mean": 0.01355451624840498, "step": 1701, "step_time": 73.15043566236272 }, { "clip_ratio/high_max": 0.011107504891697317, "clip_ratio/high_mean": 0.002854696416761726, "clip_ratio/low_mean": 0.0004710713437816594, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003325767786009237, "completions/clipped_ratio": 0.125, "completions/max_length": 1024.0, "completions/max_terminated_length": 978.0, "completions/mean_length": 601.1500244140625, "completions/mean_terminated_length": 540.7428588867188, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.09808465838432312, "epoch": 0.9681456200227532, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01303059235215187, "kl": 0.014804467442445457, "learning_rate": 2.918088737201365e-07, "loss": -0.0028, "num_tokens": 179355888.0, "reward": 1.725000023841858, "reward_std": 0.6788943409919739, "rewards/format_reward/mean": 0.875, "rewards/format_reward/std": 0.3349320888519287, "rewards/gpt_judge_reward/mean": 0.8500000238418579, "rewards/gpt_judge_reward/std": 0.36162030696868896, "sampling/importance_sampling_ratio/max": 1.367982029914856, "sampling/importance_sampling_ratio/mean": 0.26166439056396484, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.124972343444824, "sampling/sampling_logp_difference/mean": 0.015501610934734344, "step": 1702, "step_time": 70.29301957250573 }, { "clip_ratio/high_max": 0.008183732657926157, "clip_ratio/high_mean": 0.003054374312341679, "clip_ratio/low_mean": 0.0017210192600032315, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004775393579620868, "completions/clipped_ratio": 0.3499999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 958.0, "completions/mean_length": 771.4750366210938, "completions/mean_terminated_length": 635.5, "completions/min_length": 404.0, "completions/min_terminated_length": 404.0, "entropy": 0.12805753760039806, "epoch": 0.9687144482366326, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02682279795408249, "kl": 0.013023735722526908, "learning_rate": 2.8668941979522184e-07, "loss": -0.0589, "num_tokens": 179453395.0, "reward": 1.125, "reward_std": 0.9111137986183167, "rewards/format_reward/mean": 0.6499999761581421, "rewards/format_reward/std": 0.48304590582847595, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.6779603958129883, "sampling/importance_sampling_ratio/mean": 0.5670357942581177, "sampling/importance_sampling_ratio/min": 0.006015844643115997, "sampling/sampling_logp_difference/max": 2.4706735610961914, "sampling/sampling_logp_difference/mean": 0.015047584660351276, "step": 1703, "step_time": 70.48911177925766 }, { "clip_ratio/high_max": 0.010263981355819851, "clip_ratio/high_mean": 0.002493745880201459, "clip_ratio/low_mean": 0.0020263672195142135, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004520113143371418, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 971.0, "completions/mean_length": 716.2250366210938, "completions/mean_terminated_length": 488.7391357421875, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.13846401032060385, "epoch": 0.9692832764505119, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.044380173087120056, "kl": 0.013500748784281313, "learning_rate": 2.8156996587030717e-07, "loss": -0.0564, "num_tokens": 179551460.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.201880693435669, "sampling/importance_sampling_ratio/mean": 0.4613494277000427, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6451668739318848, "sampling/sampling_logp_difference/mean": 0.01699887029826641, "step": 1704, "step_time": 70.04217039467767 }, { "clip_ratio/high_max": 0.0021263217204250395, "clip_ratio/high_mean": 0.0005936144734732807, "clip_ratio/low_mean": 0.001293945315410383, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018875597888836637, "completions/clipped_ratio": 0.925000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 891.0, "completions/mean_length": 1007.7000122070312, "completions/mean_terminated_length": 806.6666870117188, "completions/min_length": 764.0, "completions/min_terminated_length": 764.0, "entropy": 0.0714141121134162, "epoch": 0.9698521046643913, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.01065047550946474, "kl": 0.006626650254474953, "learning_rate": 2.764505119453925e-07, "loss": -0.0017, "num_tokens": 179668080.0, "reward": 0.15000000596046448, "reward_std": 0.5334935784339905, "rewards/format_reward/mean": 0.07500000298023224, "rewards/format_reward/std": 0.26674678921699524, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 1.431626558303833, "sampling/importance_sampling_ratio/mean": 0.28281936049461365, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8282904624938965, "sampling/sampling_logp_difference/mean": 0.009545444510877132, "step": 1705, "step_time": 76.37175469310023 }, { "clip_ratio/high_max": 0.01521653198869899, "clip_ratio/high_mean": 0.004500544600887224, "clip_ratio/low_mean": 0.0008300781300931703, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0053306226909626275, "completions/clipped_ratio": 0.699999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 881.1749877929688, "completions/mean_terminated_length": 547.9166870117188, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.09598738607019186, "epoch": 0.9704209328782708, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.019192034378647804, "kl": 0.00945735239656642, "learning_rate": 2.713310580204778e-07, "loss": 0.0546, "num_tokens": 179766951.0, "reward": 0.4749999940395355, "reward_std": 0.7840558886528015, "rewards/format_reward/mean": 0.30000001192092896, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.8431200981140137, "sampling/importance_sampling_ratio/mean": 0.5447100400924683, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.453439235687256, "sampling/sampling_logp_difference/mean": 0.01079412829130888, "step": 1706, "step_time": 73.97790772188455 }, { "clip_ratio/high_max": 0.012716811150312424, "clip_ratio/high_mean": 0.004004217655165121, "clip_ratio/low_mean": 0.0011718750101863407, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00517609273083508, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 944.0, "completions/mean_length": 783.2250366210938, "completions/mean_terminated_length": 565.3809814453125, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.1289557982236147, "epoch": 0.9709897610921502, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.022539956495165825, "kl": 0.013375929207541049, "learning_rate": 2.6621160409556315e-07, "loss": -0.0068, "num_tokens": 179871944.0, "reward": 1.0, "reward_std": 0.9870962500572205, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.1388659477233887, "sampling/importance_sampling_ratio/mean": 0.4288051128387451, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.443167209625244, "sampling/sampling_logp_difference/mean": 0.016804233193397522, "step": 1707, "step_time": 74.3315700199455 }, { "clip_ratio/high_max": 0.001440226478734985, "clip_ratio/high_mean": 0.0002880452884710394, "clip_ratio/low_mean": 0.00034179688009317033, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0006298421685642097, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 902.4000244140625, "completions/mean_terminated_length": 537.6000366210938, "completions/min_length": 429.0, "completions/min_terminated_length": 429.0, "entropy": 0.08740935660898685, "epoch": 0.9715585893060296, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.023806072771549225, "kl": 0.00850437447661534, "learning_rate": 2.610921501706485e-07, "loss": -0.017, "num_tokens": 179978424.0, "reward": 0.5, "reward_std": 0.8770579695701599, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.0839321613311768, "sampling/importance_sampling_ratio/mean": 0.4833725094795227, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9995143413543701, "sampling/sampling_logp_difference/mean": 0.01065682340413332, "step": 1708, "step_time": 72.49297957099043 }, { "clip_ratio/high_max": 0.01141215383540839, "clip_ratio/high_mean": 0.0031691995973233134, "clip_ratio/low_mean": 0.0034179687208961695, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006587168318219483, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 986.7250366210938, "completions/mean_terminated_length": 874.9000244140625, "completions/min_length": 713.0, "completions/min_terminated_length": 713.0, "entropy": 0.15290801413357258, "epoch": 0.9721274175199089, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015450420789420605, "kl": 0.01204718102235347, "learning_rate": 2.559726962457338e-07, "loss": 0.0032, "num_tokens": 180081813.0, "reward": 0.5, "reward_std": 0.8770580291748047, "rewards/format_reward/mean": 0.25, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 1.9639395475387573, "sampling/importance_sampling_ratio/mean": 0.32500728964805603, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.579617977142334, "sampling/sampling_logp_difference/mean": 0.01570613496005535, "step": 1709, "step_time": 75.59269505785778 }, { "clip_ratio/high_max": 0.013119123526848853, "clip_ratio/high_mean": 0.004290332522941753, "clip_ratio/low_mean": 0.0009765624890860636, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005266894993837923, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 896.0, "completions/mean_length": 688.1749877929688, "completions/mean_terminated_length": 576.2333374023438, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.11514726281166077, "epoch": 0.9726962457337884, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.03621005266904831, "kl": 0.017042261897586286, "learning_rate": 2.5085324232081914e-07, "loss": 0.0192, "num_tokens": 180186396.0, "reward": 1.350000023841858, "reward_std": 0.8638019561767578, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.579497814178467, "sampling/importance_sampling_ratio/mean": 0.5675304532051086, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.044027805328369, "sampling/sampling_logp_difference/mean": 0.015806104987859726, "step": 1710, "step_time": 73.07859275490046 }, { "clip_ratio/high_max": 0.009918078314512968, "clip_ratio/high_mean": 0.002104681043419987, "clip_ratio/low_mean": 0.0011150283826282248, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003219709382392466, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 793.0, "completions/mean_length": 712.0, "completions/mean_terminated_length": 524.7999877929688, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.11365936696529388, "epoch": 0.9732650739476678, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.04612630605697632, "kl": 0.013578841520939022, "learning_rate": 2.4573378839590446e-07, "loss": 0.0849, "num_tokens": 180291724.0, "reward": 1.225000023841858, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613893032073975, "sampling/importance_sampling_ratio/max": 2.4455676078796387, "sampling/importance_sampling_ratio/mean": 0.6060449481010437, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.975043773651123, "sampling/sampling_logp_difference/mean": 0.015264618210494518, "step": 1711, "step_time": 70.50329218176194 }, { "clip_ratio/high_max": 0.007383199117612094, "clip_ratio/high_mean": 0.002228274999652058, "clip_ratio/low_mean": 0.002124023449141532, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00435229841968976, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 919.0, "completions/mean_length": 933.8250122070312, "completions/mean_terminated_length": 696.0909423828125, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "entropy": 0.08849020581692457, "epoch": 0.9738339021615472, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.02753164991736412, "kl": 0.00854027405148372, "learning_rate": 2.406143344709898e-07, "loss": -0.0972, "num_tokens": 180399541.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 1.9812666177749634, "sampling/importance_sampling_ratio/mean": 0.3663320541381836, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2164857387542725, "sampling/sampling_logp_difference/mean": 0.011412604711949825, "step": 1712, "step_time": 79.70136823458597 }, { "clip_ratio/high_max": 0.00746709480881691, "clip_ratio/high_mean": 0.0028596464544534683, "clip_ratio/low_mean": 0.0024961969938885886, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.005355843459255993, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 910.2250366210938, "completions/mean_terminated_length": 784.4736938476562, "completions/min_length": 433.0, "completions/min_terminated_length": 433.0, "entropy": 0.07743654446676373, "epoch": 0.9744027303754266, "frac_reward_zero_std": 0.0, "grad_norm": 0.0630645602941513, "kl": 0.011992897430900484, "learning_rate": 2.3549488054607507e-07, "loss": 0.0191, "num_tokens": 180507830.0, "reward": 0.8500000238418579, "reward_std": 0.9753368496894836, "rewards/format_reward/mean": 0.44999998807907104, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.8536298274993896, "sampling/importance_sampling_ratio/mean": 0.4722371995449066, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.7845354080200195, "sampling/sampling_logp_difference/mean": 0.011190532706677914, "step": 1713, "step_time": 75.15629529766738 }, { "clip_ratio/high_max": 0.00911765731871128, "clip_ratio/high_mean": 0.0026956549554597586, "clip_ratio/low_mean": 0.0016500656529387925, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004345720662968233, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 924.0, "completions/mean_length": 889.75, "completions/mean_terminated_length": 666.0000610351562, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.0886295116506517, "epoch": 0.9749715585893061, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029603909701108932, "kl": 0.014222475176211447, "learning_rate": 2.303754266211604e-07, "loss": 0.0168, "num_tokens": 180605460.0, "reward": 0.625, "reward_std": 0.8678739666938782, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.25, "rewards/gpt_judge_reward/std": 0.43852904438972473, "sampling/importance_sampling_ratio/max": 2.8279612064361572, "sampling/importance_sampling_ratio/mean": 0.3343360126018524, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3450098037719727, "sampling/sampling_logp_difference/mean": 0.011333434842526913, "step": 1714, "step_time": 71.56779083260335 }, { "clip_ratio/high_max": 0.013048724504187703, "clip_ratio/high_mean": 0.0037955098669044673, "clip_ratio/low_mean": 0.0008010599467525026, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004596569749992341, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 857.0, "completions/mean_length": 784.5250244140625, "completions/mean_terminated_length": 607.521728515625, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.10086631588637829, "epoch": 0.9755403868031854, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.024161435663700104, "kl": 0.014705301495268941, "learning_rate": 2.2525597269624575e-07, "loss": -0.0519, "num_tokens": 180714113.0, "reward": 1.125, "reward_std": 0.9919549226760864, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.623588800430298, "sampling/importance_sampling_ratio/mean": 0.41309261322021484, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.636373996734619, "sampling/sampling_logp_difference/mean": 0.013865582644939423, "step": 1715, "step_time": 75.6380155060906 }, { "clip_ratio/high_max": 0.014480202342383564, "clip_ratio/high_mean": 0.006316525978036225, "clip_ratio/low_mean": 0.0014404296962311491, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00775695574702695, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 804.0, "completions/mean_length": 742.4000244140625, "completions/mean_terminated_length": 554.6666870117188, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.061800539027899504, "epoch": 0.9761092150170648, "frac_reward_zero_std": 0.0, "grad_norm": 0.02158309333026409, "kl": 0.008842782583087683, "learning_rate": 2.2013651877133108e-07, "loss": 0.0591, "num_tokens": 180810825.0, "reward": 1.0750000476837158, "reward_std": 0.9442810416221619, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.4113761186599731, "sampling/importance_sampling_ratio/mean": 0.3306894898414612, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.02810001373291, "sampling/sampling_logp_difference/mean": 0.009369919076561928, "step": 1716, "step_time": 71.5833660394419 }, { "clip_ratio/high_max": 0.01384572486858815, "clip_ratio/high_mean": 0.0050100211665267125, "clip_ratio/low_mean": 0.002654924915987067, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00766494614072144, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 755.7250366210938, "completions/mean_terminated_length": 536.227294921875, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.1420389600098133, "epoch": 0.9766780432309442, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.028913963586091995, "kl": 0.025622408953495324, "learning_rate": 2.150170648464164e-07, "loss": 0.0245, "num_tokens": 180914174.0, "reward": 1.0, "reward_std": 0.9607689380645752, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.9698264598846436, "sampling/importance_sampling_ratio/mean": 0.40091460943222046, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1509008407592773, "sampling/sampling_logp_difference/mean": 0.016691027209162712, "step": 1717, "step_time": 71.67436510324478 }, { "clip_ratio/high_max": 0.013330962508916855, "clip_ratio/high_mean": 0.0030368680018000305, "clip_ratio/low_mean": 0.0013183593982830644, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00435522738553118, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 864.9000244140625, "completions/mean_terminated_length": 626.25, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "entropy": 0.07683195965364575, "epoch": 0.9772468714448237, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018807804211974144, "kl": 0.009556485805660486, "learning_rate": 2.098976109215017e-07, "loss": 0.1679, "num_tokens": 181015074.0, "reward": 0.7750000357627869, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.375, "rewards/gpt_judge_reward/std": 0.4902903437614441, "sampling/importance_sampling_ratio/max": 2.8206088542938232, "sampling/importance_sampling_ratio/mean": 0.5061852335929871, "sampling/importance_sampling_ratio/min": 0.022715136408805847, "sampling/sampling_logp_difference/max": 2.0369443893432617, "sampling/sampling_logp_difference/mean": 0.010643238201737404, "step": 1718, "step_time": 70.8743391702883 }, { "clip_ratio/high_max": 0.011368021368980408, "clip_ratio/high_mean": 0.0032792214769870043, "clip_ratio/low_mean": 0.0013183593910071068, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004597580875270069, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 871.4500122070312, "completions/mean_terminated_length": 588.1428833007812, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.08235898986458778, "epoch": 0.9778156996587031, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.011461549438536167, "kl": 0.009842286817729473, "learning_rate": 2.0477815699658704e-07, "loss": -0.0033, "num_tokens": 181121484.0, "reward": 0.699999988079071, "reward_std": 0.9660918116569519, "rewards/format_reward/mean": 0.3499999940395355, "rewards/format_reward/std": 0.48304587602615356, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.1053061485290527, "sampling/importance_sampling_ratio/mean": 0.3984997272491455, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9609599113464355, "sampling/sampling_logp_difference/mean": 0.010698567144572735, "step": 1719, "step_time": 73.31481530610472 }, { "clip_ratio/high_max": 0.012792139663361013, "clip_ratio/high_mean": 0.005012854846427217, "clip_ratio/low_mean": 0.0018554687267169356, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006868323544040322, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 911.0, "completions/mean_length": 897.8250122070312, "completions/mean_terminated_length": 727.11767578125, "completions/min_length": 425.0, "completions/min_terminated_length": 425.0, "entropy": 0.10114631522446871, "epoch": 0.9783845278725825, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029161324724555016, "kl": 0.00974255648907274, "learning_rate": 1.9965870307167236e-07, "loss": -0.0695, "num_tokens": 181231061.0, "reward": 0.7750000357627869, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.561166286468506, "sampling/importance_sampling_ratio/mean": 0.33298298716545105, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2674965858459473, "sampling/sampling_logp_difference/mean": 0.012937137857079506, "step": 1720, "step_time": 78.14607783942483 }, { "clip_ratio/high_max": 0.015299055725336075, "clip_ratio/high_mean": 0.0030598111625295132, "clip_ratio/low_mean": 0.0006347656344587449, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003694576764246449, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 828.8500366210938, "completions/mean_terminated_length": 536.125, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.09321502968668938, "epoch": 0.9789533560864618, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.00943358987569809, "kl": 0.013064574799500406, "learning_rate": 1.945392491467577e-07, "loss": 0.026, "num_tokens": 181344103.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.335956335067749, "sampling/importance_sampling_ratio/mean": 0.23028326034545898, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.148226022720337, "sampling/sampling_logp_difference/mean": 0.013255561701953411, "step": 1721, "step_time": 78.78399028093554 }, { "clip_ratio/high_max": 0.011228880961425602, "clip_ratio/high_mean": 0.0028668400191236287, "clip_ratio/low_mean": 0.001391601559589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004258441622368991, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 665.9000244140625, "completions/mean_terminated_length": 512.4285888671875, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.09556208178400993, "epoch": 0.9795221843003413, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02038775198161602, "kl": 0.012567937141284347, "learning_rate": 1.89419795221843e-07, "loss": -0.036, "num_tokens": 181448715.0, "reward": 1.399999976158142, "reward_std": 0.9281909465789795, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.699999988079071, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.1897377967834473, "sampling/importance_sampling_ratio/mean": 0.46884021162986755, "sampling/importance_sampling_ratio/min": 0.004317659884691238, "sampling/sampling_logp_difference/max": 2.9227969646453857, "sampling/sampling_logp_difference/mean": 0.014414853416383266, "step": 1722, "step_time": 71.04852097877301 }, { "clip_ratio/high_max": 0.013192590209655464, "clip_ratio/high_mean": 0.004761009418871254, "clip_ratio/low_mean": 0.003450124801020138, "clip_ratio/low_min": 0.0008544921875, "clip_ratio/region_mean": 0.008211134176235646, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 871.8250122070312, "completions/mean_terminated_length": 719.6500244140625, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 0.181379621848464, "epoch": 0.9800910125142207, "frac_reward_zero_std": 0.0, "grad_norm": 0.03788140043616295, "kl": 0.015281262807548046, "learning_rate": 1.8430034129692832e-07, "loss": -0.2266, "num_tokens": 181558188.0, "reward": 0.7750000357627869, "reward_std": 0.8619446754455566, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.396150588989258, "sampling/importance_sampling_ratio/mean": 0.44733428955078125, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8350337743759155, "sampling/sampling_logp_difference/mean": 0.018828576430678368, "step": 1723, "step_time": 74.37595481006429 }, { "clip_ratio/high_max": 0.014504057355225086, "clip_ratio/high_mean": 0.003783020219998434, "clip_ratio/low_mean": 0.002898535880376585, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006681556173134595, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 947.0, "completions/mean_length": 832.75, "completions/mean_terminated_length": 659.7142944335938, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.14820122346282005, "epoch": 0.9806598407281001, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.05180357024073601, "kl": 0.01197221118491143, "learning_rate": 1.7918088737201365e-07, "loss": 0.0661, "num_tokens": 181663802.0, "reward": 0.824999988079071, "reward_std": 0.873762845993042, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.30000001192092896, "rewards/gpt_judge_reward/std": 0.46409547328948975, "sampling/importance_sampling_ratio/max": 2.7912731170654297, "sampling/importance_sampling_ratio/mean": 0.4281271994113922, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2170791625976562, "sampling/sampling_logp_difference/mean": 0.017032036557793617, "step": 1724, "step_time": 72.43945176550187 }, { "clip_ratio/high_max": 0.013053086237050593, "clip_ratio/high_mean": 0.004273784230463207, "clip_ratio/low_mean": 0.001055981740137213, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005329765903297812, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 949.0, "completions/mean_length": 724.7750244140625, "completions/mean_terminated_length": 525.2916870117188, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.10769032035022974, "epoch": 0.9812286689419796, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013051562011241913, "kl": 0.011917581548914313, "learning_rate": 1.7406143344709898e-07, "loss": -0.0484, "num_tokens": 181761953.0, "reward": 1.149999976158142, "reward_std": 1.0012812614440918, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.574999988079071, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 1.8170859813690186, "sampling/importance_sampling_ratio/mean": 0.3357430100440979, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.468437671661377, "sampling/sampling_logp_difference/mean": 0.014408486895263195, "step": 1725, "step_time": 72.09132334496826 }, { "clip_ratio/high_max": 0.014509894768707454, "clip_ratio/high_mean": 0.004407801141496748, "clip_ratio/low_mean": 0.0017578124734427547, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006165613594930619, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 896.8250122070312, "completions/mean_terminated_length": 632.6923217773438, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.0879189483821392, "epoch": 0.981797497155859, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029157700017094612, "kl": 0.010577402194030583, "learning_rate": 1.6894197952218433e-07, "loss": 0.0425, "num_tokens": 181866282.0, "reward": 0.5250000357627869, "reward_std": 0.8161039352416992, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.20000000298023224, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.4456331729888916, "sampling/importance_sampling_ratio/mean": 0.5315550565719604, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9750310182571411, "sampling/sampling_logp_difference/mean": 0.011108266189694405, "step": 1726, "step_time": 74.59484772803262 }, { "clip_ratio/high_max": 0.010328210075385869, "clip_ratio/high_mean": 0.004812882805708796, "clip_ratio/low_mean": 0.0015827983042981941, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006395681150024757, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 740.2000122070312, "completions/mean_terminated_length": 645.6000366210938, "completions/min_length": 422.0, "completions/min_terminated_length": 422.0, "entropy": 0.09176563657820225, "epoch": 0.9823663253697383, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.03485829383134842, "kl": 0.01132573455106467, "learning_rate": 1.638225255972696e-07, "loss": -0.1702, "num_tokens": 181950298.0, "reward": 1.100000023841858, "reward_std": 0.7778998613357544, "rewards/format_reward/mean": 0.75, "rewards/format_reward/std": 0.43852904438972473, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.6497061252593994, "sampling/importance_sampling_ratio/mean": 0.45329371094703674, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9676713943481445, "sampling/sampling_logp_difference/mean": 0.013055582530796528, "step": 1727, "step_time": 70.50453914375976 }, { "clip_ratio/high_max": 0.001737835118547082, "clip_ratio/high_mean": 0.0003475670237094164, "clip_ratio/low_mean": 0.0007568359560536919, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011044029797631083, "completions/clipped_ratio": 0.9750000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 1023.5750122070312, "completions/mean_terminated_length": 1007.0, "completions/min_length": 1007.0, "completions/min_terminated_length": 1007.0, "entropy": 0.11404129303991795, "epoch": 0.9829351535836177, "frac_reward_zero_std": 0.800000011920929, "grad_norm": 0.028508659452199936, "kl": 0.00727598019875586, "learning_rate": 1.5870307167235496e-07, "loss": -0.1061, "num_tokens": 182059505.0, "reward": 0.05000000074505806, "reward_std": 0.3162277638912201, "rewards/format_reward/mean": 0.02500000037252903, "rewards/format_reward/std": 0.15811389684677124, "rewards/gpt_judge_reward/mean": 0.02500000037252903, "rewards/gpt_judge_reward/std": 0.15811389684677124, "sampling/importance_sampling_ratio/max": 1.9004563093185425, "sampling/importance_sampling_ratio/mean": 0.38554897904396057, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7142237424850464, "sampling/sampling_logp_difference/mean": 0.01177216600626707, "step": 1728, "step_time": 74.53064964688383 }, { "clip_ratio/high_max": 0.009649590792832896, "clip_ratio/high_mean": 0.002112399921315955, "clip_ratio/low_mean": 0.001489257818320766, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036016577505506575, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 710.2000122070312, "completions/mean_terminated_length": 575.7142944335938, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.12714907061308622, "epoch": 0.9835039817974972, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.04136931151151657, "kl": 0.013286870904266834, "learning_rate": 1.535836177474403e-07, "loss": -0.2009, "num_tokens": 182160897.0, "reward": 1.1750000715255737, "reward_std": 0.873762845993042, "rewards/format_reward/mean": 0.699999988079071, "rewards/format_reward/std": 0.46409547328948975, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.685901403427124, "sampling/importance_sampling_ratio/mean": 0.5423535108566284, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.227665424346924, "sampling/sampling_logp_difference/mean": 0.016756484284996986, "step": 1729, "step_time": 71.14583538216539 }, { "clip_ratio/high_max": 0.014324783172924072, "clip_ratio/high_mean": 0.0043320353724993765, "clip_ratio/low_mean": 0.0009407443358213641, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052727797301486135, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 710.2999877929688, "completions/mean_terminated_length": 559.25927734375, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.11851978860795498, "epoch": 0.9840728100113766, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.018354523926973343, "kl": 0.01879997132346034, "learning_rate": 1.4846416382252562e-07, "loss": -0.0722, "num_tokens": 182268021.0, "reward": 1.274999976158142, "reward_std": 0.9333562254905701, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.6000000238418579, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.768510103225708, "sampling/importance_sampling_ratio/mean": 0.3875027298927307, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.211345672607422, "sampling/sampling_logp_difference/mean": 0.015340620651841164, "step": 1730, "step_time": 71.57188320206478 }, { "clip_ratio/high_max": 0.017220556153915823, "clip_ratio/high_mean": 0.0054034739441704005, "clip_ratio/low_mean": 0.0021163745695957914, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0075198482372798026, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 908.0, "completions/mean_length": 717.9750366210938, "completions/mean_terminated_length": 491.7826232910156, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.11854757741093636, "epoch": 0.984641638225256, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.029447611421346664, "kl": 0.01385719166137278, "learning_rate": 1.4334470989761092e-07, "loss": -0.1279, "num_tokens": 182359388.0, "reward": 1.0, "reward_std": 0.9336995482444763, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.42500001192092896, "rewards/gpt_judge_reward/std": 0.5006406307220459, "sampling/importance_sampling_ratio/max": 2.6184277534484863, "sampling/importance_sampling_ratio/mean": 0.3992616534233093, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1797237396240234, "sampling/sampling_logp_difference/mean": 0.01478985883295536, "step": 1731, "step_time": 71.08432222087868 }, { "clip_ratio/high_max": 0.01767510292120278, "clip_ratio/high_mean": 0.0074207118304912, "clip_ratio/low_mean": 0.00294197590847034, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.010362687637098134, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 770.1000366210938, "completions/mean_terminated_length": 582.434814453125, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.1515537817031145, "epoch": 0.9852104664391353, "frac_reward_zero_std": 0.0, "grad_norm": 0.016688549891114235, "kl": 0.016020672395825386, "learning_rate": 1.3822525597269625e-07, "loss": 0.0401, "num_tokens": 182459280.0, "reward": 1.024999976158142, "reward_std": 0.9469925165176392, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.0358736515045166, "sampling/importance_sampling_ratio/mean": 0.27883851528167725, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.455543041229248, "sampling/sampling_logp_difference/mean": 0.017093414440751076, "step": 1732, "step_time": 74.82996975188144 }, { "clip_ratio/high_max": 0.01686695555690676, "clip_ratio/high_mean": 0.004536981461569667, "clip_ratio/low_mean": 0.00120455334399594, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0057415347546339035, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 810.0, "completions/mean_length": 794.4750366210938, "completions/mean_terminated_length": 450.1875, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.12044441420584917, "epoch": 0.9857792946530148, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.010387442074716091, "kl": 0.013427288155071437, "learning_rate": 1.3310580204778158e-07, "loss": -0.0251, "num_tokens": 182554699.0, "reward": 0.7250000238418579, "reward_std": 0.9604352712631226, "rewards/format_reward/mean": 0.375, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.3499999940395355, "rewards/gpt_judge_reward/std": 0.48304587602615356, "sampling/importance_sampling_ratio/max": 2.5248944759368896, "sampling/importance_sampling_ratio/mean": 0.40208879113197327, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.2870235443115234, "sampling/sampling_logp_difference/mean": 0.014016225002706051, "step": 1733, "step_time": 70.27251456608064 }, { "clip_ratio/high_max": 0.01292541396105662, "clip_ratio/high_mean": 0.003507910718326457, "clip_ratio/low_mean": 0.0009765625018189894, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00448447311646305, "completions/clipped_ratio": 0.32500001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 922.0, "completions/mean_length": 665.75, "completions/mean_terminated_length": 493.25927734375, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.1017869533970952, "epoch": 0.9863481228668942, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025358643382787704, "kl": 0.01441180903930217, "learning_rate": 1.279863481228669e-07, "loss": -0.0024, "num_tokens": 182654857.0, "reward": 1.125, "reward_std": 0.8825225830078125, "rewards/format_reward/mean": 0.675000011920929, "rewards/format_reward/std": 0.47434166073799133, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 1.7815438508987427, "sampling/importance_sampling_ratio/mean": 0.4715782701969147, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8749985694885254, "sampling/sampling_logp_difference/mean": 0.014437480829656124, "step": 1734, "step_time": 72.73984467680566 }, { "clip_ratio/high_max": 0.011267571244388819, "clip_ratio/high_mean": 0.004025934613309801, "clip_ratio/low_mean": 0.0016943846057984047, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005720319226384163, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 920.0, "completions/mean_length": 842.2250366210938, "completions/mean_terminated_length": 596.2941284179688, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.08176809828728437, "epoch": 0.9869169510807736, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.030309932306408882, "kl": 0.011113957152701914, "learning_rate": 1.2286689419795223e-07, "loss": 0.0062, "num_tokens": 182746258.0, "reward": 0.824999988079071, "reward_std": 0.984169602394104, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.5508902072906494, "sampling/importance_sampling_ratio/mean": 0.4378472864627838, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.148794174194336, "sampling/sampling_logp_difference/mean": 0.010959823615849018, "step": 1735, "step_time": 72.34856780595146 }, { "clip_ratio/high_max": 0.012925528455525637, "clip_ratio/high_mean": 0.003851989808026701, "clip_ratio/low_mean": 0.0008544922166038305, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004706481966422871, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 1024.0, "completions/max_terminated_length": 852.0, "completions/mean_length": 751.25, "completions/mean_terminated_length": 549.6521606445312, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.08373539987951517, "epoch": 0.987485779294653, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02000715397298336, "kl": 0.013431782834231853, "learning_rate": 1.1774744027303753e-07, "loss": -0.0498, "num_tokens": 182846380.0, "reward": 1.0750000476837158, "reward_std": 0.9710555076599121, "rewards/format_reward/mean": 0.574999988079071, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.9765734672546387, "sampling/importance_sampling_ratio/mean": 0.5050607919692993, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9255220890045166, "sampling/sampling_logp_difference/mean": 0.012627437710762024, "step": 1736, "step_time": 72.1129132588394 }, { "clip_ratio/high_max": 0.00820461183320731, "clip_ratio/high_mean": 0.00218089617555961, "clip_ratio/low_mean": 0.0012207031286379788, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034015992714557797, "completions/clipped_ratio": 0.824999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 832.0, "completions/mean_length": 932.5750122070312, "completions/mean_terminated_length": 501.5714416503906, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.07526865182444453, "epoch": 0.9880546075085325, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02208494022488594, "kl": 0.007625863247085363, "learning_rate": 1.1262798634812287e-07, "loss": -0.0122, "num_tokens": 182946755.0, "reward": 0.3499999940395355, "reward_std": 0.7696152329444885, "rewards/format_reward/mean": 0.17499999701976776, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 2.8421401977539062, "sampling/importance_sampling_ratio/mean": 0.5209392309188843, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.318220853805542, "sampling/sampling_logp_difference/mean": 0.008892026729881763, "step": 1737, "step_time": 72.06640827679075 }, { "clip_ratio/high_max": 0.011076477589085698, "clip_ratio/high_mean": 0.0023244658441399224, "clip_ratio/low_mean": 0.0011230468953726813, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003447512717684731, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 779.2999877929688, "completions/mean_terminated_length": 412.25, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.17933345958590508, "epoch": 0.9886234357224118, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.042787693440914154, "kl": 0.013876288663595915, "learning_rate": 1.075085324232082e-07, "loss": -0.1155, "num_tokens": 183050943.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.3547515869140625, "sampling/importance_sampling_ratio/mean": 0.4139555096626282, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6525325775146484, "sampling/sampling_logp_difference/mean": 0.01845383644104004, "step": 1738, "step_time": 70.37848457903601 }, { "clip_ratio/high_max": 0.015824410773348063, "clip_ratio/high_mean": 0.004235335043631494, "clip_ratio/low_mean": 0.003271484369179234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007506819674745202, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 863.125, "completions/mean_terminated_length": 685.3157958984375, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "entropy": 0.16303474456071854, "epoch": 0.9891922639362912, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.018029995262622833, "kl": 0.013309532427228987, "learning_rate": 1.0238907849829352e-07, "loss": 0.0309, "num_tokens": 183161868.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.0278239250183105, "sampling/importance_sampling_ratio/mean": 0.33786678314208984, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9576350450515747, "sampling/sampling_logp_difference/mean": 0.017314983531832695, "step": 1739, "step_time": 74.84435187699273 }, { "clip_ratio/high_max": 0.012131458905059844, "clip_ratio/high_mean": 0.0029375586673268117, "clip_ratio/low_mean": 0.001733398465148639, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004670957045163959, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 902.0, "completions/mean_length": 716.625, "completions/mean_terminated_length": 532.2000122070312, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.14773134142160416, "epoch": 0.9897610921501706, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.013482235372066498, "kl": 0.013361227931454778, "learning_rate": 9.726962457337885e-08, "loss": 0.0163, "num_tokens": 183269309.0, "reward": 1.100000023841858, "reward_std": 0.9281910061836243, "rewards/format_reward/mean": 0.625, "rewards/format_reward/std": 0.4902903437614441, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.5196338891983032, "sampling/importance_sampling_ratio/mean": 0.31483155488967896, "sampling/importance_sampling_ratio/min": 0.002215396845713258, "sampling/sampling_logp_difference/max": 1.8761099576950073, "sampling/sampling_logp_difference/mean": 0.01732797920703888, "step": 1740, "step_time": 71.67422894039191 }, { "clip_ratio/high_max": 0.01642999134492129, "clip_ratio/high_mean": 0.006480907730292529, "clip_ratio/low_mean": 0.001977539041035925, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008458446769509465, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 790.1749877929688, "completions/mean_terminated_length": 556.3500366210938, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.1325571397319436, "epoch": 0.9903299203640501, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.022446690127253532, "kl": 0.017500133020803332, "learning_rate": 9.215017064846416e-08, "loss": -0.0375, "num_tokens": 183378860.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 1.9811458587646484, "sampling/importance_sampling_ratio/mean": 0.5300039052963257, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1077492237091064, "sampling/sampling_logp_difference/mean": 0.015353974886238575, "step": 1741, "step_time": 74.99362725391984 }, { "clip_ratio/high_max": 0.011074439098592848, "clip_ratio/high_mean": 0.0029935821512481198, "clip_ratio/low_mean": 0.0014648437663709046, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004458425857592374, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.0, "completions/mean_length": 831.2750244140625, "completions/mean_terminated_length": 638.5499877929688, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.11174714006483555, "epoch": 0.9908987485779295, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.025351040065288544, "kl": 0.0110566386138089, "learning_rate": 8.703071672354949e-08, "loss": 0.0139, "num_tokens": 183487311.0, "reward": 0.949999988079071, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.44999998807907104, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.299633026123047, "sampling/importance_sampling_ratio/mean": 0.45662134885787964, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6644335985183716, "sampling/sampling_logp_difference/mean": 0.013717243447899818, "step": 1742, "step_time": 72.44105636072345 }, { "clip_ratio/high_max": 0.008054464386077598, "clip_ratio/high_mean": 0.002162172255339101, "clip_ratio/low_mean": 0.0028076172020519152, "clip_ratio/low_min": 0.000244140625, "clip_ratio/region_mean": 0.00496978938463144, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 930.7000122070312, "completions/mean_terminated_length": 684.727294921875, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.0983750568702817, "epoch": 0.9914675767918089, "frac_reward_zero_std": 0.0, "grad_norm": 0.020580491051077843, "kl": 0.01160607731435448, "learning_rate": 8.19112627986348e-08, "loss": 0.1042, "num_tokens": 183600995.0, "reward": 0.5, "reward_std": 0.8473185300827026, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220261812210083, "rewards/gpt_judge_reward/mean": 0.22499999403953552, "rewards/gpt_judge_reward/std": 0.42290204763412476, "sampling/importance_sampling_ratio/max": 2.2115402221679688, "sampling/importance_sampling_ratio/mean": 0.40091967582702637, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9947504997253418, "sampling/sampling_logp_difference/mean": 0.01124774944037199, "step": 1743, "step_time": 76.53914028010331 }, { "clip_ratio/high_max": 0.013405480334768072, "clip_ratio/high_mean": 0.005670028862368781, "clip_ratio/low_mean": 0.0019042969161091605, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007574325572932139, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 771.8250122070312, "completions/mean_terminated_length": 493.1052551269531, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.08062513871118426, "epoch": 0.9920364050056882, "frac_reward_zero_std": 0.0, "grad_norm": 0.01963750459253788, "kl": 0.01780408842023462, "learning_rate": 7.679180887372015e-08, "loss": -0.029, "num_tokens": 183699036.0, "reward": 0.75, "reward_std": 0.8697184324264526, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.2750000059604645, "rewards/gpt_judge_reward/std": 0.45220261812210083, "sampling/importance_sampling_ratio/max": 2.8321170806884766, "sampling/importance_sampling_ratio/mean": 0.33650046586990356, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.563006639480591, "sampling/sampling_logp_difference/mean": 0.011120939627289772, "step": 1744, "step_time": 72.0545334238559 }, { "clip_ratio/high_max": 0.011633767513558269, "clip_ratio/high_mean": 0.0035985268623335287, "clip_ratio/low_mean": 0.0025634765697759576, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006162003439385444, "completions/clipped_ratio": 0.675000011920929, "completions/max_length": 1024.0, "completions/max_terminated_length": 970.0, "completions/mean_length": 887.8250122070312, "completions/mean_terminated_length": 605.0, "completions/min_length": 407.0, "completions/min_terminated_length": 407.0, "entropy": 0.098210911732167, "epoch": 0.9926052332195677, "frac_reward_zero_std": 0.0, "grad_norm": 0.02632148377597332, "kl": 0.009525883942842484, "learning_rate": 7.167235494880546e-08, "loss": 0.0173, "num_tokens": 183795813.0, "reward": 0.6500000357627869, "reward_std": 0.9486833214759827, "rewards/format_reward/mean": 0.32499998807907104, "rewards/format_reward/std": 0.47434163093566895, "rewards/gpt_judge_reward/mean": 0.32499998807907104, "rewards/gpt_judge_reward/std": 0.47434163093566895, "sampling/importance_sampling_ratio/max": 2.993422269821167, "sampling/importance_sampling_ratio/mean": 0.5678943991661072, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.703787088394165, "sampling/sampling_logp_difference/mean": 0.011104716919362545, "step": 1745, "step_time": 69.78878094092943 }, { "clip_ratio/high_max": 0.017036411678418517, "clip_ratio/high_mean": 0.0055939061276149005, "clip_ratio/low_mean": 0.0019952493312302977, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007589155458845198, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 872.8500366210938, "completions/mean_terminated_length": 721.7000122070312, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.13175737764686346, "epoch": 0.9931740614334471, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.008946954272687435, "kl": 0.016754958662204444, "learning_rate": 6.655290102389079e-08, "loss": 0.0213, "num_tokens": 183906975.0, "reward": 0.9000000357627869, "reward_std": 0.9554163813591003, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.501215696334839, "sampling/importance_sampling_ratio/mean": 0.2672385573387146, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0374555587768555, "sampling/sampling_logp_difference/mean": 0.014815612696111202, "step": 1746, "step_time": 75.05195601773448 }, { "clip_ratio/high_max": 0.010734450479503721, "clip_ratio/high_mean": 0.00371273813652806, "clip_ratio/low_mean": 0.001928710931679234, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005641449097311124, "completions/clipped_ratio": 0.4000000059604645, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 824.9750366210938, "completions/mean_terminated_length": 692.2916870117188, "completions/min_length": 439.0, "completions/min_terminated_length": 439.0, "entropy": 0.10310172755271196, "epoch": 0.9937428896473265, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.014319578185677528, "kl": 0.01376501691993326, "learning_rate": 6.143344709897612e-08, "loss": -0.0096, "num_tokens": 184007790.0, "reward": 1.149999976158142, "reward_std": 0.9753369092941284, "rewards/format_reward/mean": 0.6000000238418579, "rewards/format_reward/std": 0.49613893032073975, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.729310989379883, "sampling/importance_sampling_ratio/mean": 0.3591611087322235, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9552125930786133, "sampling/sampling_logp_difference/mean": 0.013100147247314453, "step": 1747, "step_time": 73.31446152273566 }, { "clip_ratio/high_max": 0.01594458252657205, "clip_ratio/high_mean": 0.005922214855672792, "clip_ratio/low_mean": 0.0024414062645519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008363621076568961, "completions/clipped_ratio": 0.5250000357627869, "completions/max_length": 1024.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 794.6000366210938, "completions/mean_terminated_length": 541.0526123046875, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.09567076805979013, "epoch": 0.9943117178612059, "frac_reward_zero_std": 0.0, "grad_norm": 0.029801689088344574, "kl": 0.012121118837967515, "learning_rate": 5.631399317406144e-08, "loss": 0.0945, "num_tokens": 184101598.0, "reward": 0.949999988079071, "reward_std": 1.0114727020263672, "rewards/format_reward/mean": 0.4749999940395355, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.0725691318511963, "sampling/importance_sampling_ratio/mean": 0.4831165373325348, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3105931282043457, "sampling/sampling_logp_difference/mean": 0.011684659868478775, "step": 1748, "step_time": 70.22395979426801 }, { "clip_ratio/high_max": 0.016131896409206092, "clip_ratio/high_mean": 0.0049998855974990875, "clip_ratio/low_mean": 0.002124023449141532, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007123909017536789, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 820.4249877929688, "completions/mean_terminated_length": 653.8636474609375, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "entropy": 0.12165420409291983, "epoch": 0.9948805460750854, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.015226106159389019, "kl": 0.01445141073781997, "learning_rate": 5.119453924914676e-08, "loss": 0.117, "num_tokens": 184215303.0, "reward": 1.024999976158142, "reward_std": 0.9736924171447754, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.614330291748047, "sampling/importance_sampling_ratio/mean": 0.31082582473754883, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.43790864944458, "sampling/sampling_logp_difference/mean": 0.015060859732329845, "step": 1749, "step_time": 77.6366250354331 }, { "clip_ratio/high_max": 0.011120061506517231, "clip_ratio/high_mean": 0.003319794835988432, "clip_ratio/low_mean": 0.002099609402648639, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0054194043623283505, "completions/clipped_ratio": 0.7250000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 979.0, "completions/mean_length": 928.7750244140625, "completions/mean_terminated_length": 677.727294921875, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 0.1150969211012125, "epoch": 0.9954493742889647, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.021909277886152267, "kl": 0.01162418583407998, "learning_rate": 4.607508532423208e-08, "loss": -0.0197, "num_tokens": 184319174.0, "reward": 0.45000001788139343, "reward_std": 0.7828285098075867, "rewards/format_reward/mean": 0.2750000059604645, "rewards/format_reward/std": 0.45220258831977844, "rewards/gpt_judge_reward/mean": 0.17499999701976776, "rewards/gpt_judge_reward/std": 0.38480764627456665, "sampling/importance_sampling_ratio/max": 1.9990129470825195, "sampling/importance_sampling_ratio/mean": 0.36781105399131775, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.002950429916382, "sampling/sampling_logp_difference/mean": 0.012852449901401997, "step": 1750, "step_time": 72.68331804033369 }, { "clip_ratio/high_max": 0.014816005481407046, "clip_ratio/high_mean": 0.004571986821247265, "clip_ratio/low_mean": 0.0015869140770519152, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006158900752780028, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 772.4000244140625, "completions/mean_terminated_length": 520.7999877929688, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.10650114808231592, "epoch": 0.9960182025028441, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.027324503287672997, "kl": 0.010757462587207556, "learning_rate": 4.09556313993174e-08, "loss": 0.0527, "num_tokens": 184426526.0, "reward": 0.9750000238418579, "reward_std": 0.999679446220398, "rewards/format_reward/mean": 0.5, "rewards/format_reward/std": 0.5063697099685669, "rewards/gpt_judge_reward/mean": 0.4749999940395355, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.346944808959961, "sampling/importance_sampling_ratio/mean": 0.5189396142959595, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1510634422302246, "sampling/sampling_logp_difference/mean": 0.013052401132881641, "step": 1751, "step_time": 73.92449093190953 }, { "clip_ratio/high_max": 0.011810213443823159, "clip_ratio/high_mean": 0.002819013287080452, "clip_ratio/low_mean": 0.0018976446735905483, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004716658004326746, "completions/clipped_ratio": 0.574999988079071, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 887.8250122070312, "completions/mean_terminated_length": 703.5882568359375, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "entropy": 0.1170340571552515, "epoch": 0.9965870307167235, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.017875920981168747, "kl": 0.009134114370681345, "learning_rate": 3.583617747440273e-08, "loss": -0.0407, "num_tokens": 184538767.0, "reward": 0.5, "reward_std": 0.6405126452445984, "rewards/format_reward/mean": 0.42500001192092896, "rewards/format_reward/std": 0.5006406307220459, "rewards/gpt_judge_reward/mean": 0.07500000298023224, "rewards/gpt_judge_reward/std": 0.26674678921699524, "sampling/importance_sampling_ratio/max": 2.944342851638794, "sampling/importance_sampling_ratio/mean": 0.5644864439964294, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.6340399980545044, "sampling/sampling_logp_difference/mean": 0.012185119092464447, "step": 1752, "step_time": 75.26831029518507 }, { "clip_ratio/high_max": 0.011808665411081165, "clip_ratio/high_mean": 0.0041917031921911985, "clip_ratio/low_mean": 0.002270507837238256, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006462211051257327, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 963.0, "completions/mean_length": 794.5499877929688, "completions/mean_terminated_length": 606.8181762695312, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.1624971702694893, "epoch": 0.997155858930603, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.045681603252887726, "kl": 0.015695081208832562, "learning_rate": 3.071672354948806e-08, "loss": -0.0862, "num_tokens": 184647917.0, "reward": 1.0500000715255737, "reward_std": 0.9857965707778931, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.5, "rewards/gpt_judge_reward/std": 0.5063697099685669, "sampling/importance_sampling_ratio/max": 2.138735055923462, "sampling/importance_sampling_ratio/mean": 0.5032519698143005, "sampling/importance_sampling_ratio/min": 0.002284275135025382, "sampling/sampling_logp_difference/max": 1.8616693019866943, "sampling/sampling_logp_difference/mean": 0.017836494371294975, "step": 1753, "step_time": 74.4155563977547 }, { "clip_ratio/high_max": 0.017057158052921295, "clip_ratio/high_mean": 0.006267076591029763, "clip_ratio/low_mean": 0.0028320312849245965, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009099107992369682, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 859.7250366210938, "completions/mean_terminated_length": 711.0952758789062, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.15124933142215014, "epoch": 0.9977246871444824, "frac_reward_zero_std": 0.0, "grad_norm": 0.02083100751042366, "kl": 0.017916628974489868, "learning_rate": 2.559726962457338e-08, "loss": -0.0937, "num_tokens": 184756202.0, "reward": 1.0500000715255737, "reward_std": 1.0114725828170776, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.5249999761581421, "rewards/gpt_judge_reward/std": 0.5057362914085388, "sampling/importance_sampling_ratio/max": 2.2098336219787598, "sampling/importance_sampling_ratio/mean": 0.32518190145492554, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.699704170227051, "sampling/sampling_logp_difference/mean": 0.017203573137521744, "step": 1754, "step_time": 73.23782471544109 }, { "clip_ratio/high_max": 0.008754456066526473, "clip_ratio/high_mean": 0.002509180034394376, "clip_ratio/low_mean": 0.0008300781300931703, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00333925815357361, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 1024.0, "completions/max_terminated_length": 923.0, "completions/mean_length": 807.4000244140625, "completions/mean_terminated_length": 482.5, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.12157047912478447, "epoch": 0.9982935153583617, "frac_reward_zero_std": 0.6000000238418579, "grad_norm": 0.021124422550201416, "kl": 0.01450444053625688, "learning_rate": 2.04778156996587e-08, "loss": 0.0052, "num_tokens": 184860554.0, "reward": 0.800000011920929, "reward_std": 0.9922779202461243, "rewards/format_reward/mean": 0.4000000059604645, "rewards/format_reward/std": 0.49613896012306213, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 2.399614095687866, "sampling/importance_sampling_ratio/mean": 0.35550856590270996, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.083590507507324, "sampling/sampling_logp_difference/mean": 0.014144246466457844, "step": 1755, "step_time": 72.97690871777013 }, { "clip_ratio/high_max": 0.010527378879487514, "clip_ratio/high_mean": 0.0032724147458793595, "clip_ratio/low_mean": 0.001391601559589617, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004664016363676637, "completions/clipped_ratio": 0.4749999940395355, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 818.2750244140625, "completions/mean_terminated_length": 632.1428833007812, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.1458750255405903, "epoch": 0.9988623435722411, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.02249637246131897, "kl": 0.010997415869496763, "learning_rate": 1.535836177474403e-08, "loss": 0.0759, "num_tokens": 184973309.0, "reward": 0.925000011920929, "reward_std": 0.9442809820175171, "rewards/format_reward/mean": 0.5249999761581421, "rewards/format_reward/std": 0.5057362914085388, "rewards/gpt_judge_reward/mean": 0.4000000059604645, "rewards/gpt_judge_reward/std": 0.49613896012306213, "sampling/importance_sampling_ratio/max": 1.7661828994750977, "sampling/importance_sampling_ratio/mean": 0.26313573122024536, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.4490790367126465, "sampling/sampling_logp_difference/mean": 0.016818996518850327, "step": 1756, "step_time": 74.77799241966568 }, { "clip_ratio/high_max": 0.013086946681141853, "clip_ratio/high_mean": 0.004940110273309983, "clip_ratio/low_mean": 0.0011183740789419971, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0060584842576645315, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 1024.0, "completions/max_terminated_length": 979.0, "completions/mean_length": 628.75, "completions/mean_terminated_length": 559.0, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.15192205645143986, "epoch": 0.9994311717861206, "frac_reward_zero_std": 0.4000000059604645, "grad_norm": 0.040312185883522034, "kl": 0.015571952913887799, "learning_rate": 1.023890784982935e-08, "loss": 0.1314, "num_tokens": 185064715.0, "reward": 1.625, "reward_std": 0.7741827964782715, "rewards/format_reward/mean": 0.824999988079071, "rewards/format_reward/std": 0.38480764627456665, "rewards/gpt_judge_reward/mean": 0.800000011920929, "rewards/gpt_judge_reward/std": 0.40509578585624695, "sampling/importance_sampling_ratio/max": 2.849266767501831, "sampling/importance_sampling_ratio/mean": 0.4840216040611267, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8372516632080078, "sampling/sampling_logp_difference/mean": 0.017743265256285667, "step": 1757, "step_time": 66.50938767963089 }, { "clip_ratio/high_max": 0.011821674939710647, "clip_ratio/high_mean": 0.004982272082997952, "clip_ratio/low_mean": 0.0011962890603172127, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006178561132401228, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 852.8500366210938, "completions/mean_terminated_length": 712.8181762695312, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 0.11148750875145197, "epoch": 1.0, "frac_reward_zero_std": 0.20000000298023224, "grad_norm": 0.024300506338477135, "kl": 0.014122459688223898, "learning_rate": 5.119453924914675e-09, "loss": -0.0917, "num_tokens": 185174149.0, "reward": 1.100000023841858, "reward_std": 1.0076630115509033, "rewards/format_reward/mean": 0.550000011920929, "rewards/format_reward/std": 0.5038315057754517, "rewards/gpt_judge_reward/mean": 0.550000011920929, "rewards/gpt_judge_reward/std": 0.5038315057754517, "sampling/importance_sampling_ratio/max": 2.6352808475494385, "sampling/importance_sampling_ratio/mean": 0.5054251551628113, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5455973148345947, "sampling/sampling_logp_difference/mean": 0.014407535083591938, "step": 1758, "step_time": 74.89035579236224 } ], "logging_steps": 1, "max_steps": 1758, "num_input_tokens_seen": 185174149, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }