diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,5362 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.8740617180984154, + "eval_steps": 500, + "global_step": 280, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 101.0, + "completions/mean_length": 14.6328125, + "completions/min_length": 2.0, + "epoch": 0.0110803324099723, + "grad_norm": 1.5475432762019699, + "kl": 0.0, + "learning_rate": 2.2222222222222223e-05, + "loss": 5.587935447692871e-09, + "memory(GiB)": 77.28, + "reward": 0.25, + "reward_std": 0.7772719860076904, + "rewards/Response_no_think_reward/mean": 0.25, + "rewards/Response_no_think_reward/std": 1.4795188903808594, + "step": 1, + "train_speed(iter/s)": 0.001043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "epoch": 0.0221606648199446, + "grad_norm": 1.552548658770951, + "kl": 0.0, + "learning_rate": 4.4444444444444447e-05, + "loss": 5.587935447692871e-09, + "memory(GiB)": 78.06, + "step": 2, + "train_speed(iter/s)": 0.001737 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 101.0, + "completions/mean_length": 17.8828125, + "completions/min_length": 3.0, + "epoch": 0.0332409972299169, + "grad_norm": 1.291839688026328, + "kl": 0.0012106498143111821, + "learning_rate": 6.666666666666667e-05, + "loss": 0.0004578572406899184, + "memory(GiB)": 78.06, + "reward": 0.25, + "reward_std": 0.8250656127929688, + "rewards/Response_no_think_reward/mean": 0.25, + "rewards/Response_no_think_reward/std": 1.4634658098220825, + "step": 3, + "train_speed(iter/s)": 0.001451 + }, + { + "clip_ratio/high_max": 0.004197790374746546, + "clip_ratio/high_mean": 0.004197790374746546, + "clip_ratio/low_mean": 0.012020835361909121, + "clip_ratio/low_min": 0.012020835361909121, + "clip_ratio/region_mean": 0.01621862585307099, + "epoch": 0.0443213296398892, + "grad_norm": 1.0301802967661304, + "kl": 0.005412253500253428, + "learning_rate": 8.888888888888889e-05, + "loss": -0.005840146914124489, + "memory(GiB)": 78.06, + "step": 4, + "train_speed(iter/s)": 0.001769 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.002237339736893773, + "clip_ratio/low_min": 0.002237339736893773, + "clip_ratio/region_mean": 0.002237339736893773, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 101.0, + "completions/mean_length": 12.5546875, + "completions/min_length": 3.0, + "epoch": 0.055401662049861494, + "grad_norm": 2.027395284218205, + "kl": 0.014184385421685874, + "learning_rate": 0.00011111111111111112, + "loss": 0.0008896891959011555, + "memory(GiB)": 78.06, + "reward": 0.8515625, + "reward_std": 0.5118520259857178, + "rewards/Response_no_think_reward/mean": 0.8515625, + "rewards/Response_no_think_reward/std": 1.4147136211395264, + "step": 5, + "train_speed(iter/s)": 0.00164 + }, + { + "clip_ratio/high_max": 0.013844632252585143, + "clip_ratio/high_mean": 0.013844632252585143, + "clip_ratio/low_mean": 0.043920744908973575, + "clip_ratio/low_min": 0.043920744908973575, + "clip_ratio/region_mean": 0.05776537861675024, + "epoch": 0.0664819944598338, + "grad_norm": 6.212168397356187, + "kl": 1.7603500080003869, + "learning_rate": 0.00013333333333333334, + "loss": 0.01842591166496277, + "memory(GiB)": 79.98, + "step": 6, + "train_speed(iter/s)": 0.00185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.001959657238330692, + "clip_ratio/low_min": 0.001959657238330692, + "clip_ratio/region_mean": 0.001959657238330692, + "completions/clipped_ratio": 0.0, + "completions/max_length": 70.0, + "completions/mean_length": 10.359375, + "completions/min_length": 2.0, + "epoch": 0.07756232686980609, + "grad_norm": 3.537674599924329, + "kl": 0.33545287084416486, + "learning_rate": 0.00015555555555555556, + "loss": 0.0018048422643914819, + "memory(GiB)": 79.98, + "reward": 0.53125, + "reward_std": 0.6121620535850525, + "rewards/Response_no_think_reward/mean": 0.53125, + "rewards/Response_no_think_reward/std": 1.3685873746871948, + "step": 7, + "train_speed(iter/s)": 0.001773 + }, + { + "clip_ratio/high_max": 0.014042179333046079, + "clip_ratio/high_mean": 0.014042179333046079, + "clip_ratio/low_mean": 0.03925089433323592, + "clip_ratio/low_min": 0.03925089433323592, + "clip_ratio/region_mean": 0.053293074015527964, + "epoch": 0.0886426592797784, + "grad_norm": 2.64201486377639, + "kl": 1.295830228133127, + "learning_rate": 0.00017777777777777779, + "loss": 0.011679837480187416, + "memory(GiB)": 79.98, + "step": 8, + "train_speed(iter/s)": 0.001932 + }, + { + "clip_ratio/high_max": 0.0011001251987181604, + "clip_ratio/high_mean": 0.0011001251987181604, + "clip_ratio/low_mean": 0.008499634364852682, + "clip_ratio/low_min": 0.008499634364852682, + "clip_ratio/region_mean": 0.009599759563570842, + "completions/clipped_ratio": 0.0, + "completions/max_length": 62.0, + "completions/mean_length": 11.4375, + "completions/min_length": 2.0, + "epoch": 0.0997229916897507, + "grad_norm": 4.1929387792795065, + "kl": 2.5014250652166083, + "learning_rate": 0.0002, + "loss": 0.016260012984275818, + "memory(GiB)": 79.98, + "reward": 0.2734375, + "reward_std": 0.8412302732467651, + "rewards/Response_no_think_reward/mean": 0.2734375, + "rewards/Response_no_think_reward/std": 1.384474277496338, + "step": 9, + "train_speed(iter/s)": 0.001887 + }, + { + "clip_ratio/high_max": 0.02779325417941436, + "clip_ratio/high_mean": 0.02779325417941436, + "clip_ratio/low_mean": 0.07855538238072768, + "clip_ratio/low_min": 0.07855538238072768, + "clip_ratio/region_mean": 0.10634863609448075, + "epoch": 0.11080332409972299, + "grad_norm": 990.7609804346584, + "kl": 1.6530181597918272, + "learning_rate": 0.00019998312416333227, + "loss": 1.336751103401184, + "memory(GiB)": 79.98, + "step": 10, + "train_speed(iter/s)": 0.002015 + }, + { + "clip_ratio/high_max": 0.001354054023977369, + "clip_ratio/high_mean": 0.001354054023977369, + "clip_ratio/low_mean": 0.004174399145995267, + "clip_ratio/low_min": 0.004174399145995267, + "clip_ratio/region_mean": 0.005528453169972636, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 101.0, + "completions/mean_length": 26.453125, + "completions/min_length": 3.0, + "epoch": 0.12188365650969529, + "grad_norm": 1.115304569915306, + "kl": 0.3547552889212966, + "learning_rate": 0.00019993250234920636, + "loss": 0.005897670052945614, + "memory(GiB)": 79.98, + "reward": 0.5234375, + "reward_std": 0.876558780670166, + "rewards/Response_no_think_reward/mean": 0.5234375, + "rewards/Response_no_think_reward/std": 1.4251114130020142, + "step": 11, + "train_speed(iter/s)": 0.001964 + }, + { + "clip_ratio/high_max": 0.02347446102066897, + "clip_ratio/high_mean": 0.02347446102066897, + "clip_ratio/low_mean": 0.0428259114123648, + "clip_ratio/low_min": 0.0428259114123648, + "clip_ratio/region_mean": 0.06630037224385887, + "epoch": 0.1329639889196676, + "grad_norm": 2.183295109715178, + "kl": 0.6218942860141397, + "learning_rate": 0.00019984815164333163, + "loss": 0.007074224762618542, + "memory(GiB)": 79.98, + "step": 12, + "train_speed(iter/s)": 0.00207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.001206563669256866, + "clip_ratio/low_min": 0.001206563669256866, + "clip_ratio/region_mean": 0.001206563669256866, + "completions/clipped_ratio": 0.0, + "completions/max_length": 83.0, + "completions/mean_length": 14.0234375, + "completions/min_length": 4.0, + "epoch": 0.1440443213296399, + "grad_norm": 1.0424369292135751, + "kl": 0.40942037590866676, + "learning_rate": 0.00019973010051548275, + "loss": 0.004908258095383644, + "memory(GiB)": 79.98, + "reward": 0.5625, + "reward_std": 0.7038782835006714, + "rewards/Response_no_think_reward/mean": 0.5625, + "rewards/Response_no_think_reward/std": 1.2720495462417603, + "step": 13, + "train_speed(iter/s)": 0.002042 + }, + { + "clip_ratio/high_max": 0.017414433998055756, + "clip_ratio/high_mean": 0.017414433998055756, + "clip_ratio/low_mean": 0.028266766399610788, + "clip_ratio/low_min": 0.028266766399610788, + "clip_ratio/region_mean": 0.04568120092153549, + "epoch": 0.15512465373961218, + "grad_norm": 0.650161789807741, + "kl": 0.4890678570009186, + "learning_rate": 0.00019957838880989078, + "loss": -0.0075666941702365875, + "memory(GiB)": 79.98, + "step": 14, + "train_speed(iter/s)": 0.002135 + }, + { + "clip_ratio/high_max": 0.0018629207770572975, + "clip_ratio/high_mean": 0.0018629207770572975, + "clip_ratio/low_mean": 0.0069178942940197885, + "clip_ratio/low_min": 0.0069178942940197885, + "clip_ratio/region_mean": 0.00878081505652517, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 101.0, + "completions/mean_length": 17.1953125, + "completions/min_length": 4.0, + "epoch": 0.16620498614958448, + "grad_norm": 2.2362713456466605, + "kl": 0.23374600778333843, + "learning_rate": 0.00019939306773179497, + "loss": 0.0009349192259833217, + "memory(GiB)": 79.98, + "reward": 0.6875, + "reward_std": 0.8529120683670044, + "rewards/Response_no_think_reward/mean": 0.6875, + "rewards/Response_no_think_reward/std": 1.3265905380249023, + "step": 15, + "train_speed(iter/s)": 0.002091 + }, + { + "clip_ratio/high_max": 0.03511151310522109, + "clip_ratio/high_mean": 0.03511151310522109, + "clip_ratio/low_mean": 0.028677020454779267, + "clip_ratio/low_min": 0.028677020454779267, + "clip_ratio/region_mean": 0.06378853344358504, + "epoch": 0.1772853185595568, + "grad_norm": 3.3548685452904934, + "kl": 0.25148704896855634, + "learning_rate": 0.00019917419983016025, + "loss": 0.017475975677371025, + "memory(GiB)": 79.98, + "step": 16, + "train_speed(iter/s)": 0.002171 + }, + { + "clip_ratio/high_max": 0.000469924823846668, + "clip_ratio/high_mean": 0.000469924823846668, + "clip_ratio/low_mean": 0.0014688223309349269, + "clip_ratio/low_min": 0.0014688223309349269, + "clip_ratio/region_mean": 0.0019387471547815949, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 101.0, + "completions/mean_length": 18.2265625, + "completions/min_length": 6.0, + "epoch": 0.1883656509695291, + "grad_norm": 0.6996032916694407, + "kl": 0.138063008276049, + "learning_rate": 0.00019892185897656578, + "loss": 0.0021217623725533485, + "memory(GiB)": 79.98, + "reward": 0.5, + "reward_std": 0.5098158717155457, + "rewards/Response_no_think_reward/mean": 0.5, + "rewards/Response_no_think_reward/std": 1.4086347818374634, + "step": 17, + "train_speed(iter/s)": 0.002127 + }, + { + "clip_ratio/high_max": 0.01048101403284818, + "clip_ratio/high_mean": 0.01048101403284818, + "clip_ratio/low_mean": 0.028478411200921983, + "clip_ratio/low_min": 0.028478411200921983, + "clip_ratio/region_mean": 0.03895942587405443, + "epoch": 0.1994459833795014, + "grad_norm": 0.7437416506383379, + "kl": 0.2438321103884391, + "learning_rate": 0.00019863613034027224, + "loss": -0.007616878021508455, + "memory(GiB)": 79.98, + "step": 18, + "train_speed(iter/s)": 0.002199 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0009733444603625685, + "clip_ratio/low_min": 0.0009733444603625685, + "clip_ratio/region_mean": 0.0009733444603625685, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 101.0, + "completions/mean_length": 20.703125, + "completions/min_length": 6.0, + "epoch": 0.21052631578947367, + "grad_norm": 0.8303775863701255, + "kl": 0.09481111937202513, + "learning_rate": 0.0001983171103594755, + "loss": 0.0022672354243695736, + "memory(GiB)": 79.98, + "reward": -0.078125, + "reward_std": 0.6301807761192322, + "rewards/Response_no_think_reward/mean": -0.078125, + "rewards/Response_no_think_reward/std": 1.3666982650756836, + "step": 19, + "train_speed(iter/s)": 0.002152 + }, + { + "clip_ratio/high_max": 0.007191620621597394, + "clip_ratio/high_mean": 0.007191620621597394, + "clip_ratio/low_mean": 0.03550086636096239, + "clip_ratio/low_min": 0.03550086636096239, + "clip_ratio/region_mean": 0.0426924874773249, + "epoch": 0.22160664819944598, + "grad_norm": 1.7213808531720889, + "kl": 1.6638920252444223, + "learning_rate": 0.0001979649067087574, + "loss": 0.006861768197268248, + "memory(GiB)": 79.98, + "step": 20, + "train_speed(iter/s)": 0.002217 + }, + { + "clip_ratio/high_max": 0.0004032258002553135, + "clip_ratio/high_mean": 0.0004032258002553135, + "clip_ratio/low_mean": 0.003614576125983149, + "clip_ratio/low_min": 0.003614576125983149, + "clip_ratio/region_mean": 0.004017801926238462, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 101.0, + "completions/mean_length": 17.390625, + "completions/min_length": 6.0, + "epoch": 0.23268698060941828, + "grad_norm": 1.5808946015405074, + "kl": 0.1687323283404112, + "learning_rate": 0.00019757963826274357, + "loss": 0.003855248913168907, + "memory(GiB)": 79.98, + "reward": 0.5546875, + "reward_std": 0.8033354878425598, + "rewards/Response_no_think_reward/mean": 0.5546875, + "rewards/Response_no_think_reward/std": 1.3328590393066406, + "step": 21, + "train_speed(iter/s)": 0.002189 + }, + { + "clip_ratio/high_max": 0.012200821249280125, + "clip_ratio/high_mean": 0.012200821249280125, + "clip_ratio/low_mean": 0.04007338697556406, + "clip_ratio/low_min": 0.04007338697556406, + "clip_ratio/region_mean": 0.05227420857409015, + "epoch": 0.24376731301939059, + "grad_norm": 0.8850881940917656, + "kl": 0.48105100472457707, + "learning_rate": 0.0001971614350559814, + "loss": -0.004205920733511448, + "memory(GiB)": 79.98, + "step": 22, + "train_speed(iter/s)": 0.002248 + }, + { + "clip_ratio/high_max": 0.0008194574620574713, + "clip_ratio/high_mean": 0.0008194574620574713, + "clip_ratio/low_mean": 0.000811688310932368, + "clip_ratio/low_min": 0.000811688310932368, + "clip_ratio/region_mean": 0.0016311457729898393, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 101.0, + "completions/mean_length": 15.28125, + "completions/min_length": 6.0, + "epoch": 0.2548476454293629, + "grad_norm": 0.6954813297644332, + "kl": 0.11993603070732206, + "learning_rate": 0.0001967104382390511, + "loss": 0.001324990182183683, + "memory(GiB)": 79.98, + "reward": 0.78125, + "reward_std": 0.5077463984489441, + "rewards/Response_no_think_reward/mean": 0.78125, + "rewards/Response_no_think_reward/std": 1.3970582485198975, + "step": 23, + "train_speed(iter/s)": 0.002219 + }, + { + "clip_ratio/high_max": 0.014543175988364965, + "clip_ratio/high_mean": 0.014543175988364965, + "clip_ratio/low_mean": 0.027941336738877, + "clip_ratio/low_min": 0.027941336738877, + "clip_ratio/region_mean": 0.04248451231978834, + "epoch": 0.2659279778393352, + "grad_norm": 0.46891914008721675, + "kl": 0.11759324668673798, + "learning_rate": 0.00019622680003092503, + "loss": -0.009693928062915802, + "memory(GiB)": 79.98, + "step": 24, + "train_speed(iter/s)": 0.002273 + }, + { + "clip_ratio/high_max": 0.0006530559621751308, + "clip_ratio/high_mean": 0.0006530559621751308, + "clip_ratio/low_mean": 0.0019866162620019168, + "clip_ratio/low_min": 0.0019866162620019168, + "clip_ratio/region_mean": 0.002639672253280878, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 101.0, + "completions/mean_length": 12.9609375, + "completions/min_length": 4.0, + "epoch": 0.2770083102493075, + "grad_norm": 1.333182473379268, + "kl": 0.22027045290451497, + "learning_rate": 0.00019571068366759143, + "loss": 0.0026610023342072964, + "memory(GiB)": 79.98, + "reward": 0.4765625, + "reward_std": 0.8257243633270264, + "rewards/Response_no_think_reward/mean": 0.4765625, + "rewards/Response_no_think_reward/std": 1.3219220638275146, + "step": 25, + "train_speed(iter/s)": 0.002248 + }, + { + "clip_ratio/high_max": 0.02311275008833036, + "clip_ratio/high_mean": 0.02311275008833036, + "clip_ratio/low_mean": 0.03803046100074425, + "clip_ratio/low_min": 0.03803046100074425, + "clip_ratio/region_mean": 0.06114321056520566, + "epoch": 0.2880886426592798, + "grad_norm": 0.7053648057917502, + "kl": 0.2949459235333052, + "learning_rate": 0.0001951622633469592, + "loss": -0.015641074627637863, + "memory(GiB)": 79.98, + "step": 26, + "train_speed(iter/s)": 0.002297 + }, + { + "clip_ratio/high_max": 0.0004111842135898769, + "clip_ratio/high_mean": 0.0004111842135898769, + "clip_ratio/low_mean": 0.0027908546617254615, + "clip_ratio/low_min": 0.0027908546617254615, + "clip_ratio/region_mean": 0.0032020388753153384, + "completions/clipped_ratio": 0.0, + "completions/max_length": 85.0, + "completions/mean_length": 16.875, + "completions/min_length": 4.0, + "epoch": 0.29916897506925205, + "grad_norm": 1.544635006921068, + "kl": 0.20233443519100547, + "learning_rate": 0.00019458172417006347, + "loss": 0.0030936466064304113, + "memory(GiB)": 79.98, + "reward": 0.6875, + "reward_std": 0.9086803197860718, + "rewards/Response_no_think_reward/mean": 0.6875, + "rewards/Response_no_think_reward/std": 1.3265905380249023, + "step": 27, + "train_speed(iter/s)": 0.002272 + }, + { + "clip_ratio/high_max": 0.03639351949095726, + "clip_ratio/high_mean": 0.03639351949095726, + "clip_ratio/low_mean": 0.024818695266731083, + "clip_ratio/low_min": 0.024818695266731083, + "clip_ratio/region_mean": 0.06121221440844238, + "epoch": 0.31024930747922436, + "grad_norm": 2.0012174050572105, + "kl": 0.24312824057415128, + "learning_rate": 0.00019396926207859084, + "loss": -0.008060472086071968, + "memory(GiB)": 79.98, + "step": 28, + "train_speed(iter/s)": 0.002319 + }, + { + "clip_ratio/high_max": 0.0007382866751868278, + "clip_ratio/high_mean": 0.0007382866751868278, + "clip_ratio/low_mean": 0.0024169938114937395, + "clip_ratio/low_min": 0.0024169938114937395, + "clip_ratio/region_mean": 0.0031552804866805673, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 101.0, + "completions/mean_length": 23.4140625, + "completions/min_length": 3.0, + "epoch": 0.32132963988919666, + "grad_norm": 1.0333627248580772, + "kl": 0.19911292963661253, + "learning_rate": 0.0001933250837887457, + "loss": 0.004103388637304306, + "memory(GiB)": 79.98, + "reward": 0.6640625, + "reward_std": 0.7995060086250305, + "rewards/Response_no_think_reward/mean": 0.6640625, + "rewards/Response_no_think_reward/std": 1.4323447942733765, + "step": 29, + "train_speed(iter/s)": 0.002285 + }, + { + "clip_ratio/high_max": 0.021052728639915586, + "clip_ratio/high_mean": 0.021052728639915586, + "clip_ratio/low_mean": 0.03886253567179665, + "clip_ratio/low_min": 0.03886253567179665, + "clip_ratio/region_mean": 0.05991526402067393, + "epoch": 0.33240997229916897, + "grad_norm": 0.4744156832935017, + "kl": 0.2653088476508856, + "learning_rate": 0.00019264940672148018, + "loss": -0.014724130742251873, + "memory(GiB)": 79.98, + "step": 30, + "train_speed(iter/s)": 0.002328 + }, + { + "clip_ratio/high_max": 0.0008336337341461331, + "clip_ratio/high_mean": 0.0008336337341461331, + "clip_ratio/low_mean": 0.0028033541166223586, + "clip_ratio/low_min": 0.0028033541166223586, + "clip_ratio/region_mean": 0.0036369878507684916, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 101.0, + "completions/mean_length": 17.8125, + "completions/min_length": 3.0, + "epoch": 0.34349030470914127, + "grad_norm": 1.3971369328491496, + "kl": 0.3057649952825159, + "learning_rate": 0.0001919424589291108, + "loss": 0.0056169466115534306, + "memory(GiB)": 80.03, + "reward": 0.4296875, + "reward_std": 0.8740850687026978, + "rewards/Response_no_think_reward/mean": 0.4296875, + "rewards/Response_no_think_reward/std": 1.3555577993392944, + "step": 31, + "train_speed(iter/s)": 0.002304 + }, + { + "clip_ratio/high_max": 0.013716876972466707, + "clip_ratio/high_mean": 0.013716876972466707, + "clip_ratio/low_mean": 0.05669466912513599, + "clip_ratio/low_min": 0.05669466912513599, + "clip_ratio/region_mean": 0.07041154580656439, + "epoch": 0.3545706371191136, + "grad_norm": 0.7437646652660285, + "kl": 0.3698675720952451, + "learning_rate": 0.00019120447901834706, + "loss": -0.013283709064126015, + "memory(GiB)": 80.03, + "step": 32, + "train_speed(iter/s)": 0.002345 + }, + { + "clip_ratio/high_max": 0.0013764221512246877, + "clip_ratio/high_mean": 0.0013764221512246877, + "clip_ratio/low_mean": 0.0013627433363581076, + "clip_ratio/low_min": 0.0013627433363581076, + "clip_ratio/region_mean": 0.0027391654875827953, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 101.0, + "completions/mean_length": 24.609375, + "completions/min_length": 4.0, + "epoch": 0.3656509695290859, + "grad_norm": 0.8795930368140337, + "kl": 0.22658177139237523, + "learning_rate": 0.00019043571606975777, + "loss": 0.0019196830689907074, + "memory(GiB)": 80.03, + "reward": 0.4140625, + "reward_std": 0.8300054669380188, + "rewards/Response_no_think_reward/mean": 0.4140625, + "rewards/Response_no_think_reward/std": 1.258216142654419, + "step": 33, + "train_speed(iter/s)": 0.002323 + }, + { + "clip_ratio/high_max": 0.016442283987998962, + "clip_ratio/high_mean": 0.016442283987998962, + "clip_ratio/low_mean": 0.05633212422253564, + "clip_ratio/low_min": 0.05633212422253564, + "clip_ratio/region_mean": 0.07277440826874226, + "epoch": 0.3767313019390582, + "grad_norm": 0.5188690402432328, + "kl": 0.23683911142870784, + "learning_rate": 0.00018963642955370201, + "loss": -0.016352392733097076, + "memory(GiB)": 80.03, + "step": 34, + "train_speed(iter/s)": 0.002362 + }, + { + "clip_ratio/high_max": 0.0025397460121894255, + "clip_ratio/high_mean": 0.0025397460121894255, + "clip_ratio/low_mean": 0.00378477135382127, + "clip_ratio/low_min": 0.00378477135382127, + "clip_ratio/region_mean": 0.006324517366010696, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 101.0, + "completions/mean_length": 23.375, + "completions/min_length": 3.0, + "epoch": 0.3878116343490305, + "grad_norm": 0.7751350107093148, + "kl": 0.21053988160565495, + "learning_rate": 0.00018880688924275378, + "loss": 0.001979985274374485, + "memory(GiB)": 80.03, + "reward": 0.25, + "reward_std": 0.8051205277442932, + "rewards/Response_no_think_reward/mean": 0.25, + "rewards/Response_no_think_reward/std": 1.3101662397384644, + "step": 35, + "train_speed(iter/s)": 0.00234 + }, + { + "clip_ratio/high_max": 0.02383261898648925, + "clip_ratio/high_mean": 0.02383261898648925, + "clip_ratio/low_mean": 0.02296329999808222, + "clip_ratio/low_min": 0.02296329999808222, + "clip_ratio/region_mean": 0.046795917907729745, + "epoch": 0.3988919667590028, + "grad_norm": 0.6919043847160915, + "kl": 0.2092050458304584, + "learning_rate": 0.0001879473751206489, + "loss": -0.016633104532957077, + "memory(GiB)": 80.03, + "step": 36, + "train_speed(iter/s)": 0.002376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0003396739193703979, + "clip_ratio/low_min": 0.0003396739193703979, + "clip_ratio/region_mean": 0.0003396739193703979, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 101.0, + "completions/mean_length": 16.8828125, + "completions/min_length": 3.0, + "epoch": 0.4099722991689751, + "grad_norm": 1.0548464052821087, + "kl": 0.20961805986007676, + "learning_rate": 0.00018705817728778624, + "loss": 0.003657686058431864, + "memory(GiB)": 80.03, + "reward": 0.953125, + "reward_std": 0.6187193393707275, + "rewards/Response_no_think_reward/mean": 0.953125, + "rewards/Response_no_think_reward/std": 1.1961840391159058, + "step": 37, + "train_speed(iter/s)": 0.002358 + }, + { + "clip_ratio/high_max": 0.02387295541120693, + "clip_ratio/high_mean": 0.02387295541120693, + "clip_ratio/low_mean": 0.030282753868959844, + "clip_ratio/low_min": 0.030282753868959844, + "clip_ratio/region_mean": 0.05415570852346718, + "epoch": 0.42105263157894735, + "grad_norm": 0.49371052672691246, + "kl": 0.24616074899677187, + "learning_rate": 0.00018613959586331362, + "loss": -0.012075964361429214, + "memory(GiB)": 80.03, + "step": 38, + "train_speed(iter/s)": 0.002393 + }, + { + "clip_ratio/high_max": 0.0013275333330966532, + "clip_ratio/high_mean": 0.0013275333330966532, + "clip_ratio/low_mean": 0.0013904034567531198, + "clip_ratio/low_min": 0.0013904034567531198, + "clip_ratio/region_mean": 0.002717936789849773, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 101.0, + "completions/mean_length": 21.6015625, + "completions/min_length": 4.0, + "epoch": 0.43213296398891965, + "grad_norm": 0.972842541028031, + "kl": 0.16011726018041372, + "learning_rate": 0.00018519194088383273, + "loss": 0.001440724590793252, + "memory(GiB)": 80.03, + "reward": 0.625, + "reward_std": 0.6670520305633545, + "rewards/Response_no_think_reward/mean": 0.625, + "rewards/Response_no_think_reward/std": 1.3457428216934204, + "step": 39, + "train_speed(iter/s)": 0.002371 + }, + { + "clip_ratio/high_max": 0.03226059180451557, + "clip_ratio/high_mean": 0.03226059180451557, + "clip_ratio/low_mean": 0.03314233338460326, + "clip_ratio/low_min": 0.03314233338460326, + "clip_ratio/region_mean": 0.0654029252473265, + "epoch": 0.44321329639889195, + "grad_norm": 0.5253408277855696, + "kl": 0.18682625680230558, + "learning_rate": 0.00018421553219875658, + "loss": -0.013099671341478825, + "memory(GiB)": 80.03, + "step": 40, + "train_speed(iter/s)": 0.002404 + }, + { + "clip_ratio/high_max": 0.00029620854184031487, + "clip_ratio/high_mean": 0.00029620854184031487, + "clip_ratio/low_mean": 0.0009311849280493334, + "clip_ratio/low_min": 0.0009311849280493334, + "clip_ratio/region_mean": 0.0012273934698896483, + "completions/clipped_ratio": 0.0, + "completions/max_length": 88.0, + "completions/mean_length": 17.5625, + "completions/min_length": 5.0, + "epoch": 0.45429362880886426, + "grad_norm": 1.2919745010410586, + "kl": 0.2366366102360189, + "learning_rate": 0.00018321069936235503, + "loss": 0.001985038397833705, + "memory(GiB)": 80.03, + "reward": 0.859375, + "reward_std": 0.6107450723648071, + "rewards/Response_no_think_reward/mean": 0.859375, + "rewards/Response_no_think_reward/std": 1.4071491956710815, + "step": 41, + "train_speed(iter/s)": 0.002379 + }, + { + "clip_ratio/high_max": 0.01783788768807426, + "clip_ratio/high_mean": 0.01783788768807426, + "clip_ratio/low_mean": 0.044289187353570014, + "clip_ratio/low_min": 0.044289187353570014, + "clip_ratio/region_mean": 0.06212707597296685, + "epoch": 0.46537396121883656, + "grad_norm": 1.7719321167278534, + "kl": 0.6719344789162278, + "learning_rate": 0.0001821777815225245, + "loss": -0.008167732506990433, + "memory(GiB)": 80.03, + "step": 42, + "train_speed(iter/s)": 0.00241 + }, + { + "clip_ratio/high_max": 0.00036231885314919055, + "clip_ratio/high_mean": 0.00036231885314919055, + "clip_ratio/low_mean": 0.002238474931800738, + "clip_ratio/low_min": 0.002238474931800738, + "clip_ratio/region_mean": 0.0026007937849499285, + "completions/clipped_ratio": 0.0, + "completions/max_length": 97.0, + "completions/mean_length": 15.875, + "completions/min_length": 6.0, + "epoch": 0.47645429362880887, + "grad_norm": 1.1682612472028975, + "kl": 0.2416230053640902, + "learning_rate": 0.00018111712730632022, + "loss": 0.0018095734994858503, + "memory(GiB)": 80.03, + "reward": 0.0859375, + "reward_std": 0.39637458324432373, + "rewards/Response_no_think_reward/mean": 0.0859375, + "rewards/Response_no_think_reward/std": 0.9962713122367859, + "step": 43, + "train_speed(iter/s)": 0.002397 + }, + { + "clip_ratio/high_max": 0.007687599485507235, + "clip_ratio/high_mean": 0.007687599485507235, + "clip_ratio/low_mean": 0.042899149731965736, + "clip_ratio/low_min": 0.042899149731965736, + "clip_ratio/region_mean": 0.05058674863539636, + "epoch": 0.48753462603878117, + "grad_norm": 17.524605028595435, + "kl": 15.062655651359819, + "learning_rate": 0.00018002909470228842, + "loss": 0.09481670707464218, + "memory(GiB)": 80.03, + "step": 44, + "train_speed(iter/s)": 0.002427 + }, + { + "clip_ratio/high_max": 0.0012922932510264218, + "clip_ratio/high_mean": 0.0012922932510264218, + "clip_ratio/low_mean": 0.002554390055593103, + "clip_ratio/low_min": 0.002554390055593103, + "clip_ratio/region_mean": 0.003846683306619525, + "completions/clipped_ratio": 0.0, + "completions/max_length": 32.0, + "completions/mean_length": 8.546875, + "completions/min_length": 4.0, + "epoch": 0.4986149584487535, + "grad_norm": 1.474323820635427, + "kl": 0.7951482257340103, + "learning_rate": 0.00017891405093963938, + "loss": 0.008442065678536892, + "memory(GiB)": 80.03, + "reward": 0.171875, + "reward_std": 0.540536105632782, + "rewards/Response_no_think_reward/mean": 0.171875, + "rewards/Response_no_think_reward/std": 0.9648089408874512, + "step": 45, + "train_speed(iter/s)": 0.002416 + }, + { + "clip_ratio/high_max": 0.01782548933988437, + "clip_ratio/high_mean": 0.01782548933988437, + "clip_ratio/low_mean": 0.04517949424916878, + "clip_ratio/low_min": 0.04517949424916878, + "clip_ratio/region_mean": 0.06300498393829912, + "epoch": 0.5096952908587258, + "grad_norm": 23.18489815639863, + "kl": 20.399557466851547, + "learning_rate": 0.0001777723723643014, + "loss": 0.15100935101509094, + "memory(GiB)": 80.03, + "step": 46, + "train_speed(iter/s)": 0.002445 + }, + { + "clip_ratio/high_max": 0.0001338329748250544, + "clip_ratio/high_mean": 0.0001338329748250544, + "clip_ratio/low_mean": 0.0020056332577951252, + "clip_ratio/low_min": 0.0020056332577951252, + "clip_ratio/region_mean": 0.0021394662326201797, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 101.0, + "completions/mean_length": 11.8203125, + "completions/min_length": 4.0, + "epoch": 0.5207756232686981, + "grad_norm": 1.9101810703272282, + "kl": 0.8325624349527061, + "learning_rate": 0.0001766044443118978, + "loss": 0.01317879930138588, + "memory(GiB)": 80.03, + "reward": 0.21875, + "reward_std": 0.5618736743927002, + "rewards/Response_no_think_reward/mean": 0.21875, + "rewards/Response_no_think_reward/std": 0.9301384091377258, + "step": 47, + "train_speed(iter/s)": 0.002435 + }, + { + "clip_ratio/high_max": 0.01570215745596215, + "clip_ratio/high_mean": 0.01570215745596215, + "clip_ratio/low_mean": 0.01864259922876954, + "clip_ratio/low_min": 0.01864259922876954, + "clip_ratio/region_mean": 0.03434475651010871, + "epoch": 0.5318559556786704, + "grad_norm": 5.211743253538076, + "kl": 0.4838231955654919, + "learning_rate": 0.00017541066097768963, + "loss": 0.03044246882200241, + "memory(GiB)": 80.03, + "step": 48, + "train_speed(iter/s)": 0.002462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.002905012297560461, + "clip_ratio/low_min": 0.002905012297560461, + "clip_ratio/region_mean": 0.002905012297560461, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 101.0, + "completions/mean_length": 17.7265625, + "completions/min_length": 4.0, + "epoch": 0.5429362880886427, + "grad_norm": 0.9459153229739418, + "kl": 0.33009129445417784, + "learning_rate": 0.00017419142528352817, + "loss": 0.004404420033097267, + "memory(GiB)": 80.03, + "reward": -0.0859375, + "reward_std": 0.547653079032898, + "rewards/Response_no_think_reward/mean": -0.0859375, + "rewards/Response_no_think_reward/std": 0.9138250946998596, + "step": 49, + "train_speed(iter/s)": 0.002452 + }, + { + "clip_ratio/high_max": 0.00849696435034275, + "clip_ratio/high_mean": 0.00849696435034275, + "clip_ratio/low_mean": 0.059440263896249235, + "clip_ratio/low_min": 0.059440263896249235, + "clip_ratio/region_mean": 0.06793722766451538, + "epoch": 0.554016620498615, + "grad_norm": 1.5192885268898135, + "kl": 0.5284001431518845, + "learning_rate": 0.0001729471487418621, + "loss": -0.012666964903473854, + "memory(GiB)": 80.03, + "step": 50, + "train_speed(iter/s)": 0.002478 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0012755101779475808, + "clip_ratio/low_min": 0.0012755101779475808, + "clip_ratio/region_mean": 0.0012755101779475808, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 101.0, + "completions/mean_length": 12.640625, + "completions/min_length": 3.0, + "epoch": 0.5650969529085873, + "grad_norm": 1.7932835802898264, + "kl": 0.582448753528297, + "learning_rate": 0.00017167825131684513, + "loss": 0.008432275615632534, + "memory(GiB)": 80.03, + "reward": 0.15625, + "reward_std": 0.7085258960723877, + "rewards/Response_no_think_reward/mean": 0.15625, + "rewards/Response_no_think_reward/std": 1.0228685140609741, + "step": 51, + "train_speed(iter/s)": 0.002465 + }, + { + "clip_ratio/high_max": 0.006597792147658765, + "clip_ratio/high_mean": 0.006597792147658765, + "clip_ratio/low_mean": 0.049801092012785375, + "clip_ratio/low_min": 0.049801092012785375, + "clip_ratio/region_mean": 0.05639888462610543, + "epoch": 0.5761772853185596, + "grad_norm": 0.8422307837338449, + "kl": 0.7035694038495421, + "learning_rate": 0.00017038516128259115, + "loss": -0.011608053930103779, + "memory(GiB)": 80.03, + "step": 52, + "train_speed(iter/s)": 0.00249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0005558648990700021, + "clip_ratio/low_min": 0.0005558648990700021, + "clip_ratio/region_mean": 0.0005558648990700021, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 101.0, + "completions/mean_length": 12.5, + "completions/min_length": 4.0, + "epoch": 0.5872576177285319, + "grad_norm": 1.0211580850819229, + "kl": 0.7616109761074767, + "learning_rate": 0.00016906831507862443, + "loss": 0.005634145811200142, + "memory(GiB)": 80.03, + "reward": 0.1640625, + "reward_std": 0.70704185962677, + "rewards/Response_no_think_reward/mean": 0.1640625, + "rewards/Response_no_think_reward/std": 0.9702450633049011, + "step": 53, + "train_speed(iter/s)": 0.002477 + }, + { + "clip_ratio/high_max": 0.02744574609096162, + "clip_ratio/high_mean": 0.02744574609096162, + "clip_ratio/low_mean": 0.04221567645436153, + "clip_ratio/low_min": 0.04221567645436153, + "clip_ratio/region_mean": 0.06966142146848142, + "epoch": 0.5983379501385041, + "grad_norm": 0.5808426269905869, + "kl": 0.801087921798171, + "learning_rate": 0.00016772815716257412, + "loss": -0.01366308145225048, + "memory(GiB)": 80.03, + "step": 54, + "train_speed(iter/s)": 0.002502 + }, + { + "clip_ratio/high_max": 0.0003881987649947405, + "clip_ratio/high_mean": 0.0003881987649947405, + "clip_ratio/low_mean": 0.001932911021867767, + "clip_ratio/low_min": 0.001932911021867767, + "clip_ratio/region_mean": 0.0023211097868625075, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 101.0, + "completions/mean_length": 18.9921875, + "completions/min_length": 6.0, + "epoch": 0.6094182825484764, + "grad_norm": 1.1265096543789663, + "kl": 0.5226203491911292, + "learning_rate": 0.00016636513986016213, + "loss": 0.007094700820744038, + "memory(GiB)": 80.03, + "reward": 0.1875, + "reward_std": 0.7182328104972839, + "rewards/Response_no_think_reward/mean": 0.1875, + "rewards/Response_no_think_reward/std": 1.0480577945709229, + "step": 55, + "train_speed(iter/s)": 0.00249 + }, + { + "clip_ratio/high_max": 0.011908911721548066, + "clip_ratio/high_mean": 0.011908911721548066, + "clip_ratio/low_mean": 0.05817525731981732, + "clip_ratio/low_min": 0.05817525731981732, + "clip_ratio/region_mean": 0.07008416869211942, + "epoch": 0.6204986149584487, + "grad_norm": 0.5560920258062684, + "kl": 0.5747523186728358, + "learning_rate": 0.000164979723212536, + "loss": -0.014954826794564724, + "memory(GiB)": 80.03, + "step": 56, + "train_speed(iter/s)": 0.002513 + }, + { + "clip_ratio/high_max": 0.002692167239729315, + "clip_ratio/high_mean": 0.002692167239729315, + "clip_ratio/low_mean": 0.0006428283377317712, + "clip_ratio/low_min": 0.0006428283377317712, + "clip_ratio/region_mean": 0.003334995577461086, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 101.0, + "completions/mean_length": 16.53125, + "completions/min_length": 6.0, + "epoch": 0.631578947368421, + "grad_norm": 1.5596809103877662, + "kl": 0.6855434570461512, + "learning_rate": 0.00016357237482099684, + "loss": 0.0062956069596111774, + "memory(GiB)": 80.03, + "reward": 0.53125, + "reward_std": 0.5907745361328125, + "rewards/Response_no_think_reward/mean": 0.53125, + "rewards/Response_no_think_reward/std": 1.0790598392486572, + "step": 57, + "train_speed(iter/s)": 0.002501 + }, + { + "clip_ratio/high_max": 0.0316368987550959, + "clip_ratio/high_mean": 0.0316368987550959, + "clip_ratio/low_mean": 0.015557856269879267, + "clip_ratio/low_min": 0.015557856269879267, + "clip_ratio/region_mean": 0.04719475514139049, + "epoch": 0.6426592797783933, + "grad_norm": 0.950735686233936, + "kl": 0.5240823943167925, + "learning_rate": 0.00016214356968917648, + "loss": -0.015531505458056927, + "memory(GiB)": 80.03, + "step": 58, + "train_speed(iter/s)": 0.002524 + }, + { + "clip_ratio/high_max": 0.00011467889999039471, + "clip_ratio/high_mean": 0.00011467889999039471, + "clip_ratio/low_mean": 0.0004376750875962898, + "clip_ratio/low_min": 0.0004376750875962898, + "clip_ratio/region_mean": 0.0005523539875866845, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 101.0, + "completions/mean_length": 23.9140625, + "completions/min_length": 6.0, + "epoch": 0.6537396121883656, + "grad_norm": 1.3162721482480142, + "kl": 0.5214177745606321, + "learning_rate": 0.00016069379006271566, + "loss": 0.004921327345073223, + "memory(GiB)": 80.03, + "reward": -0.0625, + "reward_std": 0.6303451657295227, + "rewards/Response_no_think_reward/mean": -0.0625, + "rewards/Response_no_think_reward/std": 1.0019665956497192, + "step": 59, + "train_speed(iter/s)": 0.002511 + }, + { + "clip_ratio/high_max": 0.00810479320352897, + "clip_ratio/high_mean": 0.00810479320352897, + "clip_ratio/low_mean": 0.05729365168372169, + "clip_ratio/low_min": 0.05729365168372169, + "clip_ratio/region_mean": 0.06539844395592809, + "epoch": 0.6648199445983379, + "grad_norm": 0.5733548474442324, + "kl": 0.6363338100200053, + "learning_rate": 0.00015922352526649803, + "loss": -0.021113965660333633, + "memory(GiB)": 80.03, + "step": 60, + "train_speed(iter/s)": 0.002532 + }, + { + "clip_ratio/high_max": 0.0005996339968987741, + "clip_ratio/high_mean": 0.0005996339968987741, + "clip_ratio/low_mean": 0.004901745676761493, + "clip_ratio/low_min": 0.004901745676761493, + "clip_ratio/region_mean": 0.00550137969548814, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 101.0, + "completions/mean_length": 22.375, + "completions/min_length": 2.0, + "epoch": 0.6759002770083102, + "grad_norm": 1.887884786225259, + "kl": 2.07791917472332, + "learning_rate": 0.00015773327153949465, + "loss": 0.008165515027940273, + "memory(GiB)": 80.03, + "reward": 0.40625, + "reward_std": 0.5973243117332458, + "rewards/Response_no_think_reward/mean": 0.40625, + "rewards/Response_no_think_reward/std": 1.186787486076355, + "step": 61, + "train_speed(iter/s)": 0.002506 + }, + { + "clip_ratio/high_max": 0.017942053091246635, + "clip_ratio/high_mean": 0.017942053091246635, + "clip_ratio/low_mean": 0.06714771036058664, + "clip_ratio/low_min": 0.06714771036058664, + "clip_ratio/region_mean": 0.08508976292796433, + "epoch": 0.6869806094182825, + "grad_norm": 11.312330346219078, + "kl": 0.5888316835043952, + "learning_rate": 0.00015622353186727544, + "loss": 0.04619387909770012, + "memory(GiB)": 80.03, + "step": 62, + "train_speed(iter/s)": 0.002527 + }, + { + "clip_ratio/high_max": 0.00020032051543239504, + "clip_ratio/high_mean": 0.00020032051543239504, + "clip_ratio/low_mean": 0.001202335930429399, + "clip_ratio/low_min": 0.001202335930429399, + "clip_ratio/region_mean": 0.001402656445861794, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 101.0, + "completions/mean_length": 18.5859375, + "completions/min_length": 2.0, + "epoch": 0.6980609418282548, + "grad_norm": 1.9804020858052087, + "kl": 6.258792589243967, + "learning_rate": 0.00015469481581224272, + "loss": 0.014128579758107662, + "memory(GiB)": 80.03, + "reward": 0.78125, + "reward_std": 0.5936684608459473, + "rewards/Response_no_think_reward/mean": 0.78125, + "rewards/Response_no_think_reward/std": 1.235546350479126, + "step": 63, + "train_speed(iter/s)": 0.002486 + }, + { + "clip_ratio/high_max": 0.020077986438991502, + "clip_ratio/high_mean": 0.020077986438991502, + "clip_ratio/low_mean": 0.12254823022522032, + "clip_ratio/low_min": 0.12254823022522032, + "clip_ratio/region_mean": 0.142626216635108, + "epoch": 0.7091412742382271, + "grad_norm": 1.5460664241155249, + "kl": 5.241092938755173, + "learning_rate": 0.0001531476393416456, + "loss": -0.0033248686231672764, + "memory(GiB)": 80.03, + "step": 64, + "train_speed(iter/s)": 0.002506 + }, + { + "clip_ratio/high_max": 0.0006774475477868691, + "clip_ratio/high_mean": 0.0006774475477868691, + "clip_ratio/low_mean": 0.0009178321633953601, + "clip_ratio/low_min": 0.0009178321633953601, + "clip_ratio/region_mean": 0.0015952797257341444, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 101.0, + "completions/mean_length": 15.1484375, + "completions/min_length": 6.0, + "epoch": 0.7202216066481995, + "grad_norm": 1.528383760465238, + "kl": 1.888367731589824, + "learning_rate": 0.00015158252465343242, + "loss": 0.010861902497708797, + "memory(GiB)": 80.03, + "reward": 0.59375, + "reward_std": 0.5738716125488281, + "rewards/Response_no_think_reward/mean": 0.59375, + "rewards/Response_no_think_reward/std": 1.1462881565093994, + "step": 65, + "train_speed(iter/s)": 0.002444 + }, + { + "clip_ratio/high_max": 0.05604529404081404, + "clip_ratio/high_mean": 0.05604529404081404, + "clip_ratio/low_mean": 0.012011443439405411, + "clip_ratio/low_min": 0.012011443439405411, + "clip_ratio/region_mean": 0.06805673893541098, + "epoch": 0.7313019390581718, + "grad_norm": 0.9299498266912626, + "kl": 1.0373663480422692, + "learning_rate": 0.00015000000000000001, + "loss": -0.001186850480735302, + "memory(GiB)": 80.03, + "step": 66, + "train_speed(iter/s)": 0.002463 + }, + { + "clip_ratio/high_max": 0.006279462773818523, + "clip_ratio/high_mean": 0.006279462773818523, + "clip_ratio/low_mean": 0.006620214961003512, + "clip_ratio/low_min": 0.006620214961003512, + "clip_ratio/region_mean": 0.012899677676614374, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 101.0, + "completions/mean_length": 17.5234375, + "completions/min_length": 2.0, + "epoch": 0.7423822714681441, + "grad_norm": 3.0195311219652377, + "kl": 1.4996049946639687, + "learning_rate": 0.0001484005995098999, + "loss": 0.0103817880153656, + "memory(GiB)": 80.03, + "reward": 0.2421875, + "reward_std": 0.5205168724060059, + "rewards/Response_no_think_reward/mean": 0.2421875, + "rewards/Response_no_think_reward/std": 1.2595843076705933, + "step": 67, + "train_speed(iter/s)": 0.002399 + }, + { + "clip_ratio/high_max": 0.030079254298470914, + "clip_ratio/high_mean": 0.030079254298470914, + "clip_ratio/low_mean": 0.10815927106887102, + "clip_ratio/low_min": 0.10815927106887102, + "clip_ratio/region_mean": 0.1382385252509266, + "epoch": 0.7534626038781164, + "grad_norm": 7.7164901906176375, + "kl": 4.463950714329258, + "learning_rate": 0.0001467848630075608, + "loss": 0.0073772999458014965, + "memory(GiB)": 80.03, + "step": 68, + "train_speed(iter/s)": 0.002418 + }, + { + "clip_ratio/high_max": 0.0009369817780680023, + "clip_ratio/high_mean": 0.0009369817780680023, + "clip_ratio/low_mean": 0.006284750299528241, + "clip_ratio/low_min": 0.006284750299528241, + "clip_ratio/region_mean": 0.007221732055768371, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 101.0, + "completions/mean_length": 26.9296875, + "completions/min_length": 3.0, + "epoch": 0.7645429362880887, + "grad_norm": 2.034534758963297, + "kl": 2.321827916195616, + "learning_rate": 0.00014515333583108896, + "loss": 0.01671188324689865, + "memory(GiB)": 80.03, + "reward": 0.5390625, + "reward_std": 0.4874863028526306, + "rewards/Response_no_think_reward/mean": 0.5390625, + "rewards/Response_no_think_reward/std": 1.3095791339874268, + "step": 69, + "train_speed(iter/s)": 0.002373 + }, + { + "clip_ratio/high_max": 0.012032406637445092, + "clip_ratio/high_mean": 0.012032406637445092, + "clip_ratio/low_mean": 0.11593639780767262, + "clip_ratio/low_min": 0.11593639780767262, + "clip_ratio/region_mean": 0.1279688044451177, + "epoch": 0.775623268698061, + "grad_norm": 1.497770517232466, + "kl": 3.323778461664915, + "learning_rate": 0.00014350656864820733, + "loss": 0.0011269270908087492, + "memory(GiB)": 80.03, + "step": 70, + "train_speed(iter/s)": 0.002392 + }, + { + "clip_ratio/high_max": 0.000255623715929687, + "clip_ratio/high_mean": 0.000255623715929687, + "clip_ratio/low_mean": 0.005516766890650615, + "clip_ratio/low_min": 0.005516766890650615, + "clip_ratio/region_mean": 0.005772390664787963, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 101.0, + "completions/mean_length": 17.25, + "completions/min_length": 2.0, + "epoch": 0.7867036011080333, + "grad_norm": 2.3636642819394384, + "kl": 2.6716066980734468, + "learning_rate": 0.00014184511727039612, + "loss": 0.020374953746795654, + "memory(GiB)": 80.03, + "reward": 0.34375, + "reward_std": 0.5405020713806152, + "rewards/Response_no_think_reward/mean": 0.34375, + "rewards/Response_no_think_reward/std": 1.2323558330535889, + "step": 71, + "train_speed(iter/s)": 0.00236 + }, + { + "clip_ratio/high_max": 0.004156995622906834, + "clip_ratio/high_mean": 0.004156995622906834, + "clip_ratio/low_mean": 0.11334922257810831, + "clip_ratio/low_min": 0.11334922257810831, + "clip_ratio/region_mean": 0.11750621721148491, + "epoch": 0.7977839335180056, + "grad_norm": 1.5028809890146027, + "kl": 3.652272095438093, + "learning_rate": 0.00014016954246529696, + "loss": 0.00900588370859623, + "memory(GiB)": 80.03, + "step": 72, + "train_speed(iter/s)": 0.002378 + }, + { + "clip_ratio/high_max": 0.0024562697508372366, + "clip_ratio/high_mean": 0.0024562697508372366, + "clip_ratio/low_mean": 0.00900937442202121, + "clip_ratio/low_min": 0.00900937442202121, + "clip_ratio/region_mean": 0.01146564440568909, + "completions/clipped_ratio": 0.1015625, + "completions/max_length": 101.0, + "completions/mean_length": 20.9609375, + "completions/min_length": 6.0, + "epoch": 0.8088642659279779, + "grad_norm": 1.9262088024122541, + "kl": 2.068145776007441, + "learning_rate": 0.00013848040976744457, + "loss": 0.009554898366332054, + "memory(GiB)": 80.03, + "reward": 0.7734375, + "reward_std": 0.6535134315490723, + "rewards/Response_no_think_reward/mean": 0.7734375, + "rewards/Response_no_think_reward/std": 1.224518895149231, + "step": 73, + "train_speed(iter/s)": 0.002347 + }, + { + "clip_ratio/high_max": 0.05884167249314487, + "clip_ratio/high_mean": 0.05884167249314487, + "clip_ratio/low_mean": 0.014398490195162594, + "clip_ratio/low_min": 0.014398490195162594, + "clip_ratio/region_mean": 0.0732401623390615, + "epoch": 0.8199445983379502, + "grad_norm": 3.7794078536832525, + "kl": 1.1005137297324836, + "learning_rate": 0.00013677828928738934, + "loss": 0.027932219207286835, + "memory(GiB)": 80.03, + "step": 74, + "train_speed(iter/s)": 0.002365 + }, + { + "clip_ratio/high_max": 0.0020199596765451133, + "clip_ratio/high_mean": 0.0020199596765451133, + "clip_ratio/low_mean": 0.009859619050985202, + "clip_ratio/low_min": 0.009859619050985202, + "clip_ratio/region_mean": 0.0118795787129784, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 101.0, + "completions/mean_length": 19.734375, + "completions/min_length": 2.0, + "epoch": 0.8310249307479224, + "grad_norm": 2.6931881467895598, + "kl": 1.9703011065721512, + "learning_rate": 0.00013506375551927547, + "loss": 0.010023066774010658, + "memory(GiB)": 80.03, + "reward": 0.4140625, + "reward_std": 0.6804871559143066, + "rewards/Response_no_think_reward/mean": 0.4140625, + "rewards/Response_no_think_reward/std": 1.2005729675292969, + "step": 75, + "train_speed(iter/s)": 0.00233 + }, + { + "clip_ratio/high_max": 0.004052987133036368, + "clip_ratio/high_mean": 0.004052987133036368, + "clip_ratio/low_mean": 0.14585177681874484, + "clip_ratio/low_min": 0.14585177681874484, + "clip_ratio/region_mean": 0.14990476449020207, + "epoch": 0.8421052631578947, + "grad_norm": 1.4459331642351634, + "kl": 4.161040774546564, + "learning_rate": 0.00013333738714693956, + "loss": -0.005643587093800306, + "memory(GiB)": 80.03, + "step": 76, + "train_speed(iter/s)": 0.002347 + }, + { + "clip_ratio/high_max": 0.00046641789958812296, + "clip_ratio/high_mean": 0.00046641789958812296, + "clip_ratio/low_mean": 0.0038580247201025486, + "clip_ratio/low_min": 0.0038580247201025486, + "clip_ratio/region_mean": 0.0043244426196906716, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 99.0, + "completions/mean_length": 14.8828125, + "completions/min_length": 2.0, + "epoch": 0.853185595567867, + "grad_norm": 2.393092790871116, + "kl": 2.344057558570057, + "learning_rate": 0.00013159976684859527, + "loss": 0.014792806468904018, + "memory(GiB)": 80.03, + "reward": 0.8046875, + "reward_std": 0.515557050704956, + "rewards/Response_no_think_reward/mean": 0.8046875, + "rewards/Response_no_think_reward/std": 1.2862604856491089, + "step": 77, + "train_speed(iter/s)": 0.002292 + }, + { + "clip_ratio/high_max": 0.005874009046237916, + "clip_ratio/high_mean": 0.005874009046237916, + "clip_ratio/low_mean": 0.0609364231931977, + "clip_ratio/low_min": 0.0609364231931977, + "clip_ratio/region_mean": 0.0668104327050969, + "epoch": 0.8642659279778393, + "grad_norm": 0.9235795825109938, + "kl": 3.164612793829292, + "learning_rate": 0.00012985148110016947, + "loss": 0.001517204917035997, + "memory(GiB)": 80.03, + "step": 78, + "train_speed(iter/s)": 0.002309 + }, + { + "clip_ratio/high_max": 0.002277967141708359, + "clip_ratio/high_mean": 0.002277967141708359, + "clip_ratio/low_mean": 0.0037592062435578555, + "clip_ratio/low_min": 0.0037592062435578555, + "clip_ratio/region_mean": 0.006037173385266215, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 87.0, + "completions/mean_length": 15.703125, + "completions/min_length": 2.0, + "epoch": 0.8753462603878116, + "grad_norm": 1.666555946903183, + "kl": 2.2231151023879647, + "learning_rate": 0.00012809311997735696, + "loss": 0.012878447771072388, + "memory(GiB)": 80.03, + "reward": 0.546875, + "reward_std": 0.7733994126319885, + "rewards/Response_no_think_reward/mean": 0.546875, + "rewards/Response_no_think_reward/std": 1.3032931089401245, + "step": 79, + "train_speed(iter/s)": 0.002271 + }, + { + "clip_ratio/high_max": 0.03329824731918052, + "clip_ratio/high_mean": 0.03329824731918052, + "clip_ratio/low_mean": 0.017765316180884838, + "clip_ratio/low_min": 0.017765316180884838, + "clip_ratio/region_mean": 0.05106356361648068, + "epoch": 0.8864265927977839, + "grad_norm": 1.0568349375834465, + "kl": 1.923786539278808, + "learning_rate": 0.00012632527695645993, + "loss": -0.0005231135291978717, + "memory(GiB)": 80.03, + "step": 80, + "train_speed(iter/s)": 0.002287 + }, + { + "clip_ratio/high_max": 0.00021551724057644606, + "clip_ratio/high_mean": 0.00021551724057644606, + "clip_ratio/low_mean": 0.006801646784879267, + "clip_ratio/low_min": 0.006801646784879267, + "clip_ratio/region_mean": 0.007017164025455713, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 90.0, + "completions/mean_length": 14.03125, + "completions/min_length": 2.0, + "epoch": 0.8975069252077562, + "grad_norm": 9.477113742654934, + "kl": 3.0111945159733295, + "learning_rate": 0.00012454854871407994, + "loss": 0.016959797590970993, + "memory(GiB)": 80.03, + "reward": 0.2578125, + "reward_std": 0.5107755661010742, + "rewards/Response_no_think_reward/mean": 0.2578125, + "rewards/Response_no_think_reward/std": 1.2874077558517456, + "step": 81, + "train_speed(iter/s)": 0.002263 + }, + { + "clip_ratio/high_max": 0.012387449765810743, + "clip_ratio/high_mean": 0.012387449765810743, + "clip_ratio/low_mean": 0.04806764563545585, + "clip_ratio/low_min": 0.04806764563545585, + "clip_ratio/region_mean": 0.06045509548857808, + "epoch": 0.9085872576177285, + "grad_norm": 1.1639747912015215, + "kl": 3.7243148013949394, + "learning_rate": 0.00012276353492572935, + "loss": 0.0026693246327340603, + "memory(GiB)": 80.03, + "step": 82, + "train_speed(iter/s)": 0.002279 + }, + { + "clip_ratio/high_max": 0.0004032258002553135, + "clip_ratio/high_mean": 0.0004032258002553135, + "clip_ratio/low_mean": 0.0061736139468848705, + "clip_ratio/low_min": 0.0061736139468848705, + "clip_ratio/region_mean": 0.006576839747140184, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 99.0, + "completions/mean_length": 12.625, + "completions/min_length": 2.0, + "epoch": 0.9196675900277008, + "grad_norm": 7.345333737975268, + "kl": 1.6264500059187412, + "learning_rate": 0.00012097083806343103, + "loss": 0.02225027047097683, + "memory(GiB)": 80.03, + "reward": 0.6640625, + "reward_std": 0.4172249436378479, + "rewards/Response_no_think_reward/mean": 0.6640625, + "rewards/Response_no_think_reward/std": 1.1588573455810547, + "step": 83, + "train_speed(iter/s)": 0.00227 + }, + { + "clip_ratio/high_max": 0.0035271961241960526, + "clip_ratio/high_mean": 0.0035271961241960526, + "clip_ratio/low_mean": 0.02831025089835748, + "clip_ratio/low_min": 0.02831025089835748, + "clip_ratio/region_mean": 0.03183744702255353, + "epoch": 0.9307479224376731, + "grad_norm": 84.3494363093827, + "kl": 10.616167868487537, + "learning_rate": 0.00011917106319237386, + "loss": 0.17172452807426453, + "memory(GiB)": 80.03, + "step": 84, + "train_speed(iter/s)": 0.002286 + }, + { + "clip_ratio/high_max": 0.0011701860348694026, + "clip_ratio/high_mean": 0.0011701860348694026, + "clip_ratio/low_mean": 0.014238029951229692, + "clip_ratio/low_min": 0.014238029951229692, + "clip_ratio/region_mean": 0.015408215986099094, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 101.0, + "completions/mean_length": 19.7578125, + "completions/min_length": 4.0, + "epoch": 0.9418282548476454, + "grad_norm": 1.9907484000781221, + "kl": 2.995624510163907, + "learning_rate": 0.00011736481776669306, + "loss": 0.01750401221215725, + "memory(GiB)": 80.03, + "reward": 0.4765625, + "reward_std": 0.4854952394962311, + "rewards/Response_no_think_reward/mean": 0.4765625, + "rewards/Response_no_think_reward/std": 1.1080580949783325, + "step": 85, + "train_speed(iter/s)": 0.002274 + }, + { + "clip_ratio/high_max": 0.008182557401596569, + "clip_ratio/high_mean": 0.008182557401596569, + "clip_ratio/low_mean": 0.11124554229900241, + "clip_ratio/low_min": 0.11124554229900241, + "clip_ratio/region_mean": 0.11942810016626026, + "epoch": 0.9529085872576177, + "grad_norm": 1.9564641583381899, + "kl": 4.675610944104847, + "learning_rate": 0.00011555271142444433, + "loss": 0.009754904545843601, + "memory(GiB)": 80.03, + "step": 86, + "train_speed(iter/s)": 0.002289 + }, + { + "clip_ratio/high_max": 0.0004139889351790771, + "clip_ratio/high_mean": 0.0004139889351790771, + "clip_ratio/low_mean": 0.009557914454489946, + "clip_ratio/low_min": 0.009557914454489946, + "clip_ratio/region_mean": 0.0099719034624286, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 101.0, + "completions/mean_length": 17.90625, + "completions/min_length": 2.0, + "epoch": 0.96398891966759, + "grad_norm": 1.805689670049186, + "kl": 3.4809365491382778, + "learning_rate": 0.00011373535578184082, + "loss": 0.0213579460978508, + "memory(GiB)": 80.03, + "reward": 0.5859375, + "reward_std": 0.5313136577606201, + "rewards/Response_no_think_reward/mean": 0.5859375, + "rewards/Response_no_think_reward/std": 1.1939964294433594, + "step": 87, + "train_speed(iter/s)": 0.002277 + }, + { + "clip_ratio/high_max": 0.009385127894347534, + "clip_ratio/high_mean": 0.009385127894347534, + "clip_ratio/low_mean": 0.08647578035015613, + "clip_ratio/low_min": 0.08647578035015613, + "clip_ratio/region_mean": 0.095860909903422, + "epoch": 0.9750692520775623, + "grad_norm": 2.411018383110019, + "kl": 4.761912747140741, + "learning_rate": 0.00011191336422682237, + "loss": 0.01895134523510933, + "memory(GiB)": 80.03, + "step": 88, + "train_speed(iter/s)": 0.002291 + }, + { + "clip_ratio/high_max": 0.0031819915457163006, + "clip_ratio/high_mean": 0.0031819915457163006, + "clip_ratio/low_mean": 0.012216789647936821, + "clip_ratio/low_min": 0.012216789647936821, + "clip_ratio/region_mean": 0.015398780989926308, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 64.0, + "completions/mean_length": 11.28125, + "completions/min_length": 2.0, + "epoch": 0.9861495844875346, + "grad_norm": 2.2626010793409574, + "kl": 6.020935451146215, + "learning_rate": 0.00011008735171202684, + "loss": 0.031282562762498856, + "memory(GiB)": 80.03, + "reward": 0.2890625, + "reward_std": 0.847247302532196, + "rewards/Response_no_think_reward/mean": 0.2890625, + "rewards/Response_no_think_reward/std": 1.1915208101272583, + "step": 89, + "train_speed(iter/s)": 0.002254 + }, + { + "clip_ratio/high_max": 0.024421937006991357, + "clip_ratio/high_mean": 0.024421937006991357, + "clip_ratio/low_mean": 0.0521851975354366, + "clip_ratio/low_min": 0.0521851975354366, + "clip_ratio/region_mean": 0.07660713430959731, + "epoch": 0.997229916897507, + "grad_norm": 1.8302536272850396, + "kl": 6.024846433196217, + "learning_rate": 0.00010825793454723325, + "loss": 0.017346249893307686, + "memory(GiB)": 80.03, + "step": 90, + "train_speed(iter/s)": 0.002268 + }, + { + "clip_ratio/high_max": 0.0011776478204410523, + "clip_ratio/high_mean": 0.0011776478204410523, + "clip_ratio/low_mean": 0.0035608798498287797, + "clip_ratio/low_min": 0.0035608798498287797, + "clip_ratio/region_mean": 0.004738527670269832, + "completions/clipped_ratio": 0.125, + "completions/max_length": 101.0, + "completions/mean_length": 24.046875, + "completions/min_length": 3.0, + "epoch": 1.0110803324099722, + "grad_norm": 2.3698869054826233, + "kl": 5.333947827733937, + "learning_rate": 0.00010642573019134703, + "loss": 0.028505954891443253, + "memory(GiB)": 80.03, + "reward": 0.84375, + "reward_std": 0.6655995845794678, + "rewards/Response_no_think_reward/mean": 0.84375, + "rewards/Response_no_think_reward/std": 1.2065274715423584, + "step": 91, + "train_speed(iter/s)": 0.002232 + }, + { + "clip_ratio/high_max": 0.02221274602925405, + "clip_ratio/high_mean": 0.02221274602925405, + "clip_ratio/low_mean": 0.04682085904642008, + "clip_ratio/low_min": 0.04682085904642008, + "clip_ratio/region_mean": 0.0690336050465703, + "epoch": 1.0221606648199446, + "grad_norm": 1.377879165623118, + "kl": 6.767704317186144, + "learning_rate": 0.00010459135704399718, + "loss": 0.01659482903778553, + "memory(GiB)": 80.03, + "step": 92, + "train_speed(iter/s)": 0.002246 + }, + { + "clip_ratio/high_max": 0.0007944914977997541, + "clip_ratio/high_mean": 0.0007944914977997541, + "clip_ratio/low_mean": 0.005214237666223198, + "clip_ratio/low_min": 0.005214237666223198, + "clip_ratio/region_mean": 0.006008729164022952, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 79.0, + "completions/mean_length": 11.1953125, + "completions/min_length": 2.0, + "epoch": 1.0332409972299168, + "grad_norm": 1.128328936062302, + "kl": 2.6099998716963455, + "learning_rate": 0.00010275543423681621, + "loss": 0.01443527452647686, + "memory(GiB)": 80.03, + "reward": 0.5703125, + "reward_std": 0.34169840812683105, + "rewards/Response_no_think_reward/mean": 0.5703125, + "rewards/Response_no_think_reward/std": 1.26543128490448, + "step": 93, + "train_speed(iter/s)": 0.00223 + }, + { + "clip_ratio/high_max": 0.024202606233302504, + "clip_ratio/high_mean": 0.024202606233302504, + "clip_ratio/low_mean": 0.017650849069468677, + "clip_ratio/low_min": 0.017650849069468677, + "clip_ratio/region_mean": 0.04185345536097884, + "epoch": 1.0443213296398892, + "grad_norm": 0.5979565041663649, + "kl": 1.9247902451315895, + "learning_rate": 0.00010091858142447265, + "loss": 0.005481676664203405, + "memory(GiB)": 80.03, + "step": 94, + "train_speed(iter/s)": 0.002243 + }, + { + "clip_ratio/high_max": 0.002601411077193916, + "clip_ratio/high_mean": 0.002601411077193916, + "clip_ratio/low_mean": 0.005088170932140201, + "clip_ratio/low_min": 0.005088170932140201, + "clip_ratio/region_mean": 0.007689581951126456, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 101.0, + "completions/mean_length": 18.25, + "completions/min_length": 4.0, + "epoch": 1.0554016620498614, + "grad_norm": 2.0648308983451917, + "kl": 1.713204285595566, + "learning_rate": 9.908141857552737e-05, + "loss": 0.015094820410013199, + "memory(GiB)": 80.03, + "reward": 0.8046875, + "reward_std": 0.47176384925842285, + "rewards/Response_no_think_reward/mean": 0.8046875, + "rewards/Response_no_think_reward/std": 1.1708977222442627, + "step": 95, + "train_speed(iter/s)": 0.002221 + }, + { + "clip_ratio/high_max": 0.01881888063508086, + "clip_ratio/high_mean": 0.01881888063508086, + "clip_ratio/low_mean": 0.026617751631420106, + "clip_ratio/low_min": 0.026617751631420106, + "clip_ratio/region_mean": 0.0454366315389052, + "epoch": 1.0664819944598338, + "grad_norm": 1.5544793796965792, + "kl": 1.6157679219031706, + "learning_rate": 9.724456576318381e-05, + "loss": 0.012739625759422779, + "memory(GiB)": 80.03, + "step": 96, + "train_speed(iter/s)": 0.002234 + }, + { + "clip_ratio/high_max": 0.0020559211261570454, + "clip_ratio/high_mean": 0.0020559211261570454, + "clip_ratio/low_mean": 0.0043267360888421535, + "clip_ratio/low_min": 0.0043267360888421535, + "clip_ratio/region_mean": 0.006382657098583877, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 41.0, + "completions/mean_length": 11.046875, + "completions/min_length": 3.0, + "epoch": 1.077562326869806, + "grad_norm": 2.5531547462941115, + "kl": 1.4817758100107312, + "learning_rate": 9.540864295600283e-05, + "loss": 0.016522858291864395, + "memory(GiB)": 80.03, + "reward": 0.9453125, + "reward_std": 0.4848037362098694, + "rewards/Response_no_think_reward/mean": 0.9453125, + "rewards/Response_no_think_reward/std": 1.1454023122787476, + "step": 97, + "train_speed(iter/s)": 0.00222 + }, + { + "clip_ratio/high_max": 0.01358306163456291, + "clip_ratio/high_mean": 0.01358306163456291, + "clip_ratio/low_mean": 0.019928007503040135, + "clip_ratio/low_min": 0.019928007503040135, + "clip_ratio/region_mean": 0.03351106960326433, + "epoch": 1.0886426592797784, + "grad_norm": 1.0799722223140724, + "kl": 1.3886900492943823, + "learning_rate": 9.357426980865301e-05, + "loss": -0.0007159767556004226, + "memory(GiB)": 80.03, + "step": 98, + "train_speed(iter/s)": 0.002234 + }, + { + "clip_ratio/high_max": 0.001544758939417079, + "clip_ratio/high_mean": 0.001544758939417079, + "clip_ratio/low_mean": 0.0037943847200949676, + "clip_ratio/low_min": 0.0037943847200949676, + "clip_ratio/region_mean": 0.005339143652236089, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 101.0, + "completions/mean_length": 18.546875, + "completions/min_length": 3.0, + "epoch": 1.0997229916897506, + "grad_norm": 0.943375664401446, + "kl": 1.7392279328778386, + "learning_rate": 9.174206545276677e-05, + "loss": 0.014223725534975529, + "memory(GiB)": 80.03, + "reward": 1.1015625, + "reward_std": 0.39179152250289917, + "rewards/Response_no_think_reward/mean": 1.1015625, + "rewards/Response_no_think_reward/std": 1.2221051454544067, + "step": 99, + "train_speed(iter/s)": 0.002219 + }, + { + "clip_ratio/high_max": 0.01209607784403488, + "clip_ratio/high_mean": 0.01209607784403488, + "clip_ratio/low_mean": 0.011367922488716431, + "clip_ratio/low_min": 0.011367922488716431, + "clip_ratio/region_mean": 0.023464000318199396, + "epoch": 1.110803324099723, + "grad_norm": 0.8253425068817393, + "kl": 1.7898913251701742, + "learning_rate": 8.991264828797319e-05, + "loss": 0.006411677226424217, + "memory(GiB)": 80.03, + "step": 100, + "train_speed(iter/s)": 0.002232 + }, + { + "clip_ratio/high_max": 0.0038593837525695562, + "clip_ratio/high_mean": 0.0038593837525695562, + "clip_ratio/low_mean": 0.007897521747509018, + "clip_ratio/low_min": 0.007897521747509018, + "clip_ratio/region_mean": 0.011756905500078574, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 51.0, + "completions/mean_length": 18.0390625, + "completions/min_length": 2.0, + "epoch": 1.1218836565096952, + "grad_norm": 2.1091313211648033, + "kl": 4.433779507409781, + "learning_rate": 8.808663577317764e-05, + "loss": 0.026584401726722717, + "memory(GiB)": 77.36, + "reward": 0.234375, + "reward_std": 0.353938490152359, + "rewards/Response_no_think_reward_1/mean": 0.234375, + "rewards/Response_no_think_reward_1/std": 0.8275223970413208, + "step": 101, + "train_speed(iter/s)": 0.112917 + }, + { + "clip_ratio/high_max": 0.0257808348396793, + "clip_ratio/high_mean": 0.0257808348396793, + "clip_ratio/low_mean": 0.013182859780499712, + "clip_ratio/low_min": 0.013182859780499712, + "clip_ratio/region_mean": 0.038963694794801995, + "epoch": 1.1329639889196677, + "grad_norm": 0.8069647208167382, + "kl": 2.7774715912528336, + "learning_rate": 8.626464421815919e-05, + "loss": 0.008937789127230644, + "memory(GiB)": 77.36, + "step": 102, + "train_speed(iter/s)": 0.09385 + }, + { + "clip_ratio/high_max": 0.00028669723542407155, + "clip_ratio/high_mean": 0.00028669723542407155, + "clip_ratio/low_mean": 0.0008802816737443209, + "clip_ratio/low_min": 0.0008802816737443209, + "clip_ratio/region_mean": 0.0011669789091683924, + "completions/clipped_ratio": 0.109375, + "completions/max_length": 51.0, + "completions/mean_length": 18.140625, + "completions/min_length": 3.0, + "epoch": 1.1440443213296398, + "grad_norm": 3.4014714828505634, + "kl": 1.8993340344168246, + "learning_rate": 8.444728857555572e-05, + "loss": 0.029605647549033165, + "memory(GiB)": 77.47, + "reward": 0.390625, + "reward_std": 0.1173202246427536, + "rewards/Response_no_think_reward_1/mean": 0.390625, + "rewards/Response_no_think_reward_1/std": 0.8440096974372864, + "step": 103, + "train_speed(iter/s)": 0.053798 + }, + { + "clip_ratio/high_max": 0.0013847328373230994, + "clip_ratio/high_mean": 0.0013847328373230994, + "clip_ratio/low_mean": 0.004156515002250671, + "clip_ratio/low_min": 0.004156515002250671, + "clip_ratio/region_mean": 0.00554124778136611, + "epoch": 1.1551246537396123, + "grad_norm": 0.6574286317590833, + "kl": 0.8905834904871881, + "learning_rate": 8.263518223330697e-05, + "loss": 0.007299074437469244, + "memory(GiB)": 77.47, + "step": 104, + "train_speed(iter/s)": 0.049358 + }, + { + "clip_ratio/high_max": 0.00036549707874655724, + "clip_ratio/high_mean": 0.00036549707874655724, + "clip_ratio/low_mean": 0.003542276710504666, + "clip_ratio/low_min": 0.003542276710504666, + "clip_ratio/region_mean": 0.003907773789251223, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 51.0, + "completions/mean_length": 12.1015625, + "completions/min_length": 2.0, + "epoch": 1.1662049861495845, + "grad_norm": 1.418255621232663, + "kl": 1.0295969531871378, + "learning_rate": 8.082893680762619e-05, + "loss": 0.008070322684943676, + "memory(GiB)": 77.47, + "reward": 0.5625, + "reward_std": 0.283821702003479, + "rewards/Response_no_think_reward_1/mean": 0.5625, + "rewards/Response_no_think_reward_1/std": 0.6728714108467102, + "step": 105, + "train_speed(iter/s)": 0.035507 + }, + { + "clip_ratio/high_max": 0.015719514689408243, + "clip_ratio/high_mean": 0.015719514689408243, + "clip_ratio/low_mean": 0.010580109432339668, + "clip_ratio/low_min": 0.010580109432339668, + "clip_ratio/region_mean": 0.02629962412174791, + "epoch": 1.1772853185595569, + "grad_norm": 0.9124403050854863, + "kl": 0.933376073371619, + "learning_rate": 7.902916193656898e-05, + "loss": -0.002763347467407584, + "memory(GiB)": 79.43, + "step": 106, + "train_speed(iter/s)": 0.033633 + }, + { + "clip_ratio/high_max": 0.0023960003745742142, + "clip_ratio/high_mean": 0.0023960003745742142, + "clip_ratio/low_mean": 0.002585217938758433, + "clip_ratio/low_min": 0.002585217938758433, + "clip_ratio/region_mean": 0.004981218371540308, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 51.0, + "completions/mean_length": 13.6328125, + "completions/min_length": 2.0, + "epoch": 1.188365650969529, + "grad_norm": 2.0082146476980807, + "kl": 1.3074679019264295, + "learning_rate": 7.72364650742707e-05, + "loss": 0.011914699338376522, + "memory(GiB)": 79.43, + "reward": 0.328125, + "reward_std": 0.18394747376441956, + "rewards/Response_no_think_reward_1/mean": 0.328125, + "rewards/Response_no_think_reward_1/std": 0.7110973596572876, + "step": 107, + "train_speed(iter/s)": 0.027655 + }, + { + "clip_ratio/high_max": 0.00717889575753361, + "clip_ratio/high_mean": 0.00717889575753361, + "clip_ratio/low_mean": 0.008150914101861417, + "clip_ratio/low_min": 0.008150914101861417, + "clip_ratio/region_mean": 0.015329810208640993, + "epoch": 1.1994459833795015, + "grad_norm": 0.5916941576021421, + "kl": 1.0833495813399168, + "learning_rate": 7.54514512859201e-05, + "loss": 0.0037035662680864334, + "memory(GiB)": 79.43, + "step": 108, + "train_speed(iter/s)": 0.026591 + }, + { + "clip_ratio/high_max": 0.004251995822414756, + "clip_ratio/high_mean": 0.004251995822414756, + "clip_ratio/low_mean": 0.008190131688024849, + "clip_ratio/low_min": 0.008190131688024849, + "clip_ratio/region_mean": 0.012442127510439605, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 51.0, + "completions/mean_length": 12.234375, + "completions/min_length": 4.0, + "epoch": 1.2105263157894737, + "grad_norm": 2.6396182071410137, + "kl": 2.141993957106024, + "learning_rate": 7.36747230435401e-05, + "loss": 0.011151588521897793, + "memory(GiB)": 79.43, + "reward": -0.2265625, + "reward_std": 0.3407740592956543, + "rewards/Response_no_think_reward_1/mean": -0.2265625, + "rewards/Response_no_think_reward_1/std": 0.6425201892852783, + "step": 109, + "train_speed(iter/s)": 0.022574 + }, + { + "clip_ratio/high_max": 0.004035328107420355, + "clip_ratio/high_mean": 0.004035328107420355, + "clip_ratio/low_mean": 0.08847818686626852, + "clip_ratio/low_min": 0.08847818686626852, + "clip_ratio/region_mean": 0.0925135153811425, + "epoch": 1.221606648199446, + "grad_norm": 1.4171762109533736, + "kl": 3.1860878374427557, + "learning_rate": 7.190688002264308e-05, + "loss": 0.0037668771110475063, + "memory(GiB)": 79.43, + "step": 110, + "train_speed(iter/s)": 0.021907 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.002568919211626053, + "clip_ratio/low_min": 0.002568919211626053, + "clip_ratio/region_mean": 0.002568919211626053, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 48.0, + "completions/mean_length": 11.7734375, + "completions/min_length": 5.0, + "epoch": 1.2326869806094183, + "grad_norm": 1.02177770841713, + "kl": 1.6274185269139707, + "learning_rate": 7.014851889983057e-05, + "loss": 0.010478168725967407, + "memory(GiB)": 79.43, + "reward": 0.453125, + "reward_std": 0.3291260004043579, + "rewards/Response_no_think_reward_1/mean": 0.453125, + "rewards/Response_no_think_reward_1/std": 0.685730516910553, + "step": 111, + "train_speed(iter/s)": 0.019127 + }, + { + "clip_ratio/high_max": 0.013096909533487633, + "clip_ratio/high_mean": 0.013096909533487633, + "clip_ratio/low_mean": 0.016107605304569006, + "clip_ratio/low_min": 0.016107605304569006, + "clip_ratio/region_mean": 0.029204514692537487, + "epoch": 1.2437673130193905, + "grad_norm": 0.79999454502468, + "kl": 1.5544351362623274, + "learning_rate": 6.840023315140475e-05, + "loss": 0.0022247314918786287, + "memory(GiB)": 79.43, + "step": 112, + "train_speed(iter/s)": 0.018681 + }, + { + "clip_ratio/high_max": 0.0008340688509633765, + "clip_ratio/high_mean": 0.0008340688509633765, + "clip_ratio/low_mean": 0.007082634954713285, + "clip_ratio/low_min": 0.007082634954713285, + "clip_ratio/region_mean": 0.007916703820228577, + "completions/clipped_ratio": 0.1171875, + "completions/max_length": 51.0, + "completions/mean_length": 13.53125, + "completions/min_length": 3.0, + "epoch": 1.254847645429363, + "grad_norm": 1.6246389101003882, + "kl": 1.6449745513964444, + "learning_rate": 6.666261285306047e-05, + "loss": 0.014917208813130856, + "memory(GiB)": 79.43, + "reward": 0.1953125, + "reward_std": 0.2754020392894745, + "rewards/Response_no_think_reward_1/mean": 0.1953125, + "rewards/Response_no_think_reward_1/std": 0.869958758354187, + "step": 113, + "train_speed(iter/s)": 0.016592 + }, + { + "clip_ratio/high_max": 0.006006928611896001, + "clip_ratio/high_mean": 0.006006928611896001, + "clip_ratio/low_mean": 0.039706501411274076, + "clip_ratio/low_min": 0.039706501411274076, + "clip_ratio/region_mean": 0.045713430270552635, + "epoch": 1.2659279778393353, + "grad_norm": 0.940807595373605, + "kl": 1.8106578167062253, + "learning_rate": 6.493624448072457e-05, + "loss": 0.006363555323332548, + "memory(GiB)": 79.43, + "step": 114, + "train_speed(iter/s)": 0.016278 + }, + { + "clip_ratio/high_max": 0.0052188277477398515, + "clip_ratio/high_mean": 0.0052188277477398515, + "clip_ratio/low_mean": 0.004236701555782929, + "clip_ratio/low_min": 0.004236701555782929, + "clip_ratio/region_mean": 0.009455529390834272, + "completions/clipped_ratio": 0.1796875, + "completions/max_length": 51.0, + "completions/mean_length": 14.3671875, + "completions/min_length": 2.0, + "epoch": 1.2770083102493075, + "grad_norm": 2.8040352101197437, + "kl": 5.506896490347572, + "learning_rate": 6.322171071261071e-05, + "loss": 0.03229736536741257, + "memory(GiB)": 79.43, + "reward": 0.296875, + "reward_std": 0.43980443477630615, + "rewards/Response_no_think_reward_1/mean": 0.296875, + "rewards/Response_no_think_reward_1/std": 0.8905397057533264, + "step": 115, + "train_speed(iter/s)": 0.014636 + }, + { + "clip_ratio/high_max": 0.01615347486222163, + "clip_ratio/high_mean": 0.01615347486222163, + "clip_ratio/low_mean": 0.015428576618432999, + "clip_ratio/low_min": 0.015428576618432999, + "clip_ratio/region_mean": 0.031582050723955035, + "epoch": 1.2880886426592797, + "grad_norm": 1.8650893219420759, + "kl": 4.258469146443531, + "learning_rate": 6.151959023255545e-05, + "loss": 0.020270783454179764, + "memory(GiB)": 79.43, + "step": 116, + "train_speed(iter/s)": 0.014407 + }, + { + "clip_ratio/high_max": 0.005145640461705625, + "clip_ratio/high_mean": 0.005145640461705625, + "clip_ratio/low_mean": 0.008849590638419613, + "clip_ratio/low_min": 0.008849590638419613, + "clip_ratio/region_mean": 0.013995230983709916, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 51.0, + "completions/mean_length": 16.3125, + "completions/min_length": 2.0, + "epoch": 1.299168975069252, + "grad_norm": 2.4581301952944186, + "kl": 3.745300827547908, + "learning_rate": 5.983045753470308e-05, + "loss": 0.024165078997612, + "memory(GiB)": 79.43, + "reward": 0.453125, + "reward_std": 0.5826787948608398, + "rewards/Response_no_think_reward_1/mean": 0.453125, + "rewards/Response_no_think_reward_1/std": 0.8405039310455322, + "step": 117, + "train_speed(iter/s)": 0.013208 + }, + { + "clip_ratio/high_max": 0.04523819196037948, + "clip_ratio/high_mean": 0.04523819196037948, + "clip_ratio/low_mean": 0.01510127488290891, + "clip_ratio/low_min": 0.01510127488290891, + "clip_ratio/region_mean": 0.060339466377627105, + "epoch": 1.3102493074792243, + "grad_norm": 2.131099568305027, + "kl": 2.847630614414811, + "learning_rate": 5.8154882729603876e-05, + "loss": 0.01335166022181511, + "memory(GiB)": 79.43, + "step": 118, + "train_speed(iter/s)": 0.013037 + }, + { + "clip_ratio/high_max": 0.002125054510543123, + "clip_ratio/high_mean": 0.002125054510543123, + "clip_ratio/low_mean": 0.0127813016588334, + "clip_ratio/low_min": 0.0127813016588334, + "clip_ratio/region_mean": 0.0149063560529612, + "completions/clipped_ratio": 0.203125, + "completions/max_length": 51.0, + "completions/mean_length": 19.59375, + "completions/min_length": 3.0, + "epoch": 1.3213296398891967, + "grad_norm": 2.690385409772627, + "kl": 2.0104650848079473, + "learning_rate": 5.64934313517927e-05, + "loss": 0.01620793715119362, + "memory(GiB)": 79.43, + "reward": 0.2578125, + "reward_std": 0.41504764556884766, + "rewards/Response_no_think_reward_1/mean": 0.2578125, + "rewards/Response_no_think_reward_1/std": 0.8625734448432922, + "step": 119, + "train_speed(iter/s)": 0.011541 + }, + { + "clip_ratio/high_max": 0.0026382115320302546, + "clip_ratio/high_mean": 0.0026382115320302546, + "clip_ratio/low_mean": 0.07074824426672421, + "clip_ratio/low_min": 0.07074824426672421, + "clip_ratio/region_mean": 0.07338645646814257, + "epoch": 1.332409972299169, + "grad_norm": 1.3611711034111147, + "kl": 2.366683575557545, + "learning_rate": 5.484666416891109e-05, + "loss": 0.00427972199395299, + "memory(GiB)": 79.43, + "step": 120, + "train_speed(iter/s)": 0.011424 + }, + { + "clip_ratio/high_max": 0.00034340660204179585, + "clip_ratio/high_mean": 0.00034340660204179585, + "clip_ratio/low_mean": 0.0057576168910600245, + "clip_ratio/low_min": 0.0057576168910600245, + "clip_ratio/region_mean": 0.00610102349310182, + "completions/clipped_ratio": 0.109375, + "completions/max_length": 51.0, + "completions/mean_length": 14.1640625, + "completions/min_length": 3.0, + "epoch": 1.3434903047091413, + "grad_norm": 1.7730415769710828, + "kl": 2.253111455589533, + "learning_rate": 5.321513699243924e-05, + "loss": 0.015828199684619904, + "memory(GiB)": 79.43, + "reward": 0.2578125, + "reward_std": 0.2504267692565918, + "rewards/Response_no_think_reward_1/mean": 0.2578125, + "rewards/Response_no_think_reward_1/std": 0.8533961176872253, + "step": 121, + "train_speed(iter/s)": 0.010746 + }, + { + "clip_ratio/high_max": 0.002522797236451879, + "clip_ratio/high_mean": 0.002522797236451879, + "clip_ratio/low_mean": 0.03892370511312038, + "clip_ratio/low_min": 0.03892370511312038, + "clip_ratio/region_mean": 0.041446502320468426, + "epoch": 1.3545706371191135, + "grad_norm": 0.9270755733265906, + "kl": 2.852388353087008, + "learning_rate": 5.159940049010015e-05, + "loss": 0.00514911487698555, + "memory(GiB)": 79.43, + "step": 122, + "train_speed(iter/s)": 0.010652 + }, + { + "clip_ratio/high_max": 0.0008753835718380287, + "clip_ratio/high_mean": 0.0008753835718380287, + "clip_ratio/low_mean": 0.0012499999720603228, + "clip_ratio/low_min": 0.0012499999720603228, + "clip_ratio/region_mean": 0.0021253835438983515, + "completions/clipped_ratio": 0.1171875, + "completions/max_length": 51.0, + "completions/mean_length": 15.8515625, + "completions/min_length": 4.0, + "epoch": 1.365650969529086, + "grad_norm": 24.163527329310554, + "kl": 24.496757203305606, + "learning_rate": 5.000000000000002e-05, + "loss": 0.23566541075706482, + "memory(GiB)": 79.43, + "reward": 0.3984375, + "reward_std": 0.3253360986709595, + "rewards/Response_no_think_reward_1/mean": 0.3984375, + "rewards/Response_no_think_reward_1/std": 0.6912255883216858, + "step": 123, + "train_speed(iter/s)": 0.010181 + }, + { + "clip_ratio/high_max": 0.0009191176941385493, + "clip_ratio/high_mean": 0.0009191176941385493, + "clip_ratio/low_mean": 0.025084137567318976, + "clip_ratio/low_min": 0.025084137567318976, + "clip_ratio/region_mean": 0.026003255392424762, + "epoch": 1.3767313019390581, + "grad_norm": 4.61798364138243, + "kl": 5.482510100933723, + "learning_rate": 4.841747534656763e-05, + "loss": 0.03564080968499184, + "memory(GiB)": 79.43, + "step": 124, + "train_speed(iter/s)": 0.010103 + }, + { + "clip_ratio/high_max": 0.0004921259824186563, + "clip_ratio/high_mean": 0.0004921259824186563, + "clip_ratio/low_mean": 0.0009282178361900151, + "clip_ratio/low_min": 0.0009282178361900151, + "clip_ratio/region_mean": 0.0014203438186086714, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 51.0, + "completions/mean_length": 16.671875, + "completions/min_length": 3.0, + "epoch": 1.3878116343490305, + "grad_norm": 1.7230796893481761, + "kl": 1.7308420957997441, + "learning_rate": 4.685236065835443e-05, + "loss": 0.01634293608367443, + "memory(GiB)": 79.43, + "reward": 0.171875, + "reward_std": 0.13258251547813416, + "rewards/Response_no_think_reward_1/mean": 0.171875, + "rewards/Response_no_think_reward_1/std": 0.8971465826034546, + "step": 125, + "train_speed(iter/s)": 0.009672 + }, + { + "clip_ratio/high_max": 0.0003094059356953949, + "clip_ratio/high_mean": 0.0003094059356953949, + "clip_ratio/low_mean": 0.02542525064200163, + "clip_ratio/low_min": 0.02542525064200163, + "clip_ratio/region_mean": 0.025734656490385532, + "epoch": 1.3988919667590027, + "grad_norm": 0.6685767852950313, + "kl": 2.058195663616061, + "learning_rate": 4.530518418775733e-05, + "loss": 0.008037411607801914, + "memory(GiB)": 79.43, + "step": 126, + "train_speed(iter/s)": 0.009605 + }, + { + "clip_ratio/high_max": 0.0035992932971566916, + "clip_ratio/high_mean": 0.0035992932971566916, + "clip_ratio/low_mean": 0.0026174120721407235, + "clip_ratio/low_min": 0.0026174120721407235, + "clip_ratio/region_mean": 0.006216705543920398, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 51.0, + "completions/mean_length": 13.1015625, + "completions/min_length": 3.0, + "epoch": 1.4099722991689752, + "grad_norm": 0.898899557372063, + "kl": 1.8244512832025066, + "learning_rate": 4.3776468132724604e-05, + "loss": 0.013885595835745335, + "memory(GiB)": 79.43, + "reward": 0.703125, + "reward_std": 0.1841355264186859, + "rewards/Response_no_think_reward_1/mean": 0.703125, + "rewards/Response_no_think_reward_1/std": 0.552152156829834, + "step": 127, + "train_speed(iter/s)": 0.009212 + }, + { + "clip_ratio/high_max": 0.00787301326636225, + "clip_ratio/high_mean": 0.00787301326636225, + "clip_ratio/low_mean": 0.003833794384263456, + "clip_ratio/low_min": 0.003833794384263456, + "clip_ratio/region_mean": 0.011706807999871671, + "epoch": 1.4210526315789473, + "grad_norm": 0.6061431338619399, + "kl": 1.8278243900567759, + "learning_rate": 4.2266728460505375e-05, + "loss": 0.009479128755629063, + "memory(GiB)": 79.43, + "step": 128, + "train_speed(iter/s)": 0.009157 + }, + { + "clip_ratio/high_max": 0.0016208597226068377, + "clip_ratio/high_mean": 0.0016208597226068377, + "clip_ratio/low_mean": 0.004291799967177212, + "clip_ratio/low_min": 0.004291799967177212, + "clip_ratio/region_mean": 0.00591265968978405, + "completions/clipped_ratio": 0.2421875, + "completions/max_length": 51.0, + "completions/mean_length": 13.8359375, + "completions/min_length": 3.0, + "epoch": 1.4321329639889195, + "grad_norm": 4.256081630918958, + "kl": 8.371784689603373, + "learning_rate": 4.077647473350201e-05, + "loss": 0.07109890133142471, + "memory(GiB)": 79.43, + "reward": 0.1640625, + "reward_std": 0.12844271957874298, + "rewards/Response_no_think_reward_1/mean": 0.1640625, + "rewards/Response_no_think_reward_1/std": 0.9029901623725891, + "step": 129, + "train_speed(iter/s)": 0.008828 + }, + { + "clip_ratio/high_max": 0.00940803368575871, + "clip_ratio/high_mean": 0.00940803368575871, + "clip_ratio/low_mean": 0.006534474319778383, + "clip_ratio/low_min": 0.006534474319778383, + "clip_ratio/region_mean": 0.01594250788912177, + "epoch": 1.443213296398892, + "grad_norm": 2.0975168570419678, + "kl": 6.735093111405149, + "learning_rate": 3.9306209937284346e-05, + "loss": 0.05932926759123802, + "memory(GiB)": 79.43, + "step": 130, + "train_speed(iter/s)": 0.008781 + }, + { + "clip_ratio/high_max": 0.004439248572452925, + "clip_ratio/high_mean": 0.004439248572452925, + "clip_ratio/low_mean": 0.006494191708043218, + "clip_ratio/low_min": 0.006494191708043218, + "clip_ratio/region_mean": 0.01093344046967104, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 51.0, + "completions/mean_length": 13.9140625, + "completions/min_length": 3.0, + "epoch": 1.4542936288088644, + "grad_norm": 1.7347303322446779, + "kl": 3.3224903107620776, + "learning_rate": 3.7856430310823545e-05, + "loss": 0.028562916442751884, + "memory(GiB)": 79.43, + "reward": 0.2578125, + "reward_std": 0.4904649257659912, + "rewards/Response_no_think_reward_1/mean": 0.2578125, + "rewards/Response_no_think_reward_1/std": 0.8441190123558044, + "step": 131, + "train_speed(iter/s)": 0.008479 + }, + { + "clip_ratio/high_max": 0.03524596616625786, + "clip_ratio/high_mean": 0.03524596616625786, + "clip_ratio/low_mean": 0.00640316259523388, + "clip_ratio/low_min": 0.00640316259523388, + "clip_ratio/region_mean": 0.04164912860142067, + "epoch": 1.4653739612188366, + "grad_norm": 1.8414978565783118, + "kl": 2.55354578839615, + "learning_rate": 3.642762517900322e-05, + "loss": 0.016005922108888626, + "memory(GiB)": 79.43, + "step": 132, + "train_speed(iter/s)": 0.008439 + }, + { + "clip_ratio/high_max": 0.00021331057359930128, + "clip_ratio/high_mean": 0.00021331057359930128, + "clip_ratio/low_mean": 0.007616169808898121, + "clip_ratio/low_min": 0.007616169808898121, + "clip_ratio/region_mean": 0.007829480382497422, + "completions/clipped_ratio": 0.125, + "completions/max_length": 51.0, + "completions/mean_length": 16.9296875, + "completions/min_length": 4.0, + "epoch": 1.4764542936288088, + "grad_norm": 2.995948442993347, + "kl": 2.835317355929874, + "learning_rate": 3.5020276787464056e-05, + "loss": 0.020519524812698364, + "memory(GiB)": 79.43, + "reward": -0.109375, + "reward_std": 0.39537471532821655, + "rewards/Response_no_think_reward_1/mean": -0.109375, + "rewards/Response_no_think_reward_1/std": 0.7860434055328369, + "step": 133, + "train_speed(iter/s)": 0.008214 + }, + { + "clip_ratio/high_max": 0.0014211501111276448, + "clip_ratio/high_mean": 0.0014211501111276448, + "clip_ratio/low_mean": 0.017294615099672228, + "clip_ratio/low_min": 0.017294615099672228, + "clip_ratio/region_mean": 0.01871576503617689, + "epoch": 1.4875346260387812, + "grad_norm": 2.3860638079880268, + "kl": 2.9496174114756286, + "learning_rate": 3.363486013983788e-05, + "loss": 0.015305472537875175, + "memory(GiB)": 79.43, + "step": 134, + "train_speed(iter/s)": 0.008178 + }, + { + "clip_ratio/high_max": 0.004125965555431321, + "clip_ratio/high_mean": 0.004125965555431321, + "clip_ratio/low_mean": 0.01121757403598167, + "clip_ratio/low_min": 0.01121757403598167, + "clip_ratio/region_mean": 0.01534353947499767, + "completions/clipped_ratio": 0.2421875, + "completions/max_length": 51.0, + "completions/mean_length": 17.59375, + "completions/min_length": 5.0, + "epoch": 1.4986149584487536, + "grad_norm": 1.9611447116444427, + "kl": 4.596401881426573, + "learning_rate": 3.227184283742591e-05, + "loss": 0.037781622260808945, + "memory(GiB)": 79.43, + "reward": 0.140625, + "reward_std": 0.3686816394329071, + "rewards/Response_no_think_reward_1/mean": 0.140625, + "rewards/Response_no_think_reward_1/std": 0.9698962569236755, + "step": 135, + "train_speed(iter/s)": 0.007948 + }, + { + "clip_ratio/high_max": 0.003497250087093562, + "clip_ratio/high_mean": 0.003497250087093562, + "clip_ratio/low_mean": 0.0877209399768617, + "clip_ratio/low_min": 0.0877209399768617, + "clip_ratio/region_mean": 0.09121819020947441, + "epoch": 1.5096952908587258, + "grad_norm": 1.7237744503611014, + "kl": 4.757093018852174, + "learning_rate": 3.093168492137557e-05, + "loss": 0.02426687255501747, + "memory(GiB)": 79.43, + "step": 136, + "train_speed(iter/s)": 0.007917 + }, + { + "clip_ratio/high_max": 0.0014806788822170347, + "clip_ratio/high_mean": 0.0014806788822170347, + "clip_ratio/low_mean": 0.008024309470783919, + "clip_ratio/low_min": 0.008024309470783919, + "clip_ratio/region_mean": 0.009504988382104784, + "completions/clipped_ratio": 0.2265625, + "completions/max_length": 49.0, + "completions/mean_length": 15.859375, + "completions/min_length": 4.0, + "epoch": 1.520775623268698, + "grad_norm": 2.1836739109067973, + "kl": 5.872505620121956, + "learning_rate": 2.9614838717408867e-05, + "loss": 0.05089029669761658, + "memory(GiB)": 79.43, + "reward": 0.1640625, + "reward_std": 0.48505210876464844, + "rewards/Response_no_think_reward_1/mean": 0.1640625, + "rewards/Response_no_think_reward_1/std": 0.8303053379058838, + "step": 137, + "train_speed(iter/s)": 0.00773 + }, + { + "clip_ratio/high_max": 0.009889701497741044, + "clip_ratio/high_mean": 0.009889701497741044, + "clip_ratio/low_mean": 0.03272932127583772, + "clip_ratio/low_min": 0.03272932127583772, + "clip_ratio/region_mean": 0.04261902256985195, + "epoch": 1.5318559556786704, + "grad_norm": 1.660171536889304, + "kl": 5.461771305650473, + "learning_rate": 2.8321748683154893e-05, + "loss": 0.03914007171988487, + "memory(GiB)": 79.43, + "step": 138, + "train_speed(iter/s)": 0.007702 + }, + { + "clip_ratio/high_max": 0.0019176136120222509, + "clip_ratio/high_mean": 0.0019176136120222509, + "clip_ratio/low_mean": 0.007726973562967032, + "clip_ratio/low_min": 0.007726973562967032, + "clip_ratio/region_mean": 0.009644587058573961, + "completions/clipped_ratio": 0.265625, + "completions/max_length": 51.0, + "completions/mean_length": 15.359375, + "completions/min_length": 4.0, + "epoch": 1.5429362880886428, + "grad_norm": 3.7132212896004106, + "kl": 4.60741166153457, + "learning_rate": 2.7052851258137935e-05, + "loss": 0.039852242916822433, + "memory(GiB)": 79.43, + "reward": 0.0078125, + "reward_std": 0.3704521656036377, + "rewards/Response_no_think_reward_1/mean": 0.0078125, + "rewards/Response_no_think_reward_1/std": 0.7985823154449463, + "step": 139, + "train_speed(iter/s)": 0.007537 + }, + { + "clip_ratio/high_max": 0.0055117253214120865, + "clip_ratio/high_mean": 0.0055117253214120865, + "clip_ratio/low_mean": 0.016618184628896415, + "clip_ratio/low_min": 0.016618184628896415, + "clip_ratio/region_mean": 0.022129910183139145, + "epoch": 1.554016620498615, + "grad_norm": 2.5249297141615665, + "kl": 4.39259727431272, + "learning_rate": 2.5808574716471856e-05, + "loss": 0.03180728852748871, + "memory(GiB)": 79.43, + "step": 140, + "train_speed(iter/s)": 0.007513 + }, + { + "clip_ratio/high_max": 0.001243597303982824, + "clip_ratio/high_mean": 0.001243597303982824, + "clip_ratio/low_mean": 0.009276433673221618, + "clip_ratio/low_min": 0.009276433673221618, + "clip_ratio/region_mean": 0.010520030977204442, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 51.0, + "completions/mean_length": 13.2578125, + "completions/min_length": 4.0, + "epoch": 1.5650969529085872, + "grad_norm": 2.0040824091240594, + "kl": 6.036298241931945, + "learning_rate": 2.4589339022310386e-05, + "loss": 0.044014573097229004, + "memory(GiB)": 79.43, + "reward": 0.2109375, + "reward_std": 0.3419404625892639, + "rewards/Response_no_think_reward_1/mean": 0.2109375, + "rewards/Response_no_think_reward_1/std": 0.7902191877365112, + "step": 141, + "train_speed(iter/s)": 0.007347 + }, + { + "clip_ratio/high_max": 0.008325316943228245, + "clip_ratio/high_mean": 0.008325316943228245, + "clip_ratio/low_mean": 0.024790967552689835, + "clip_ratio/low_min": 0.024790967552689835, + "clip_ratio/region_mean": 0.0331162846123334, + "epoch": 1.5761772853185596, + "grad_norm": 1.8697222079345355, + "kl": 6.215300239622593, + "learning_rate": 2.339555568810221e-05, + "loss": 0.03644668310880661, + "memory(GiB)": 79.43, + "step": 142, + "train_speed(iter/s)": 0.007326 + }, + { + "clip_ratio/high_max": 0.0029302738257683814, + "clip_ratio/high_mean": 0.0029302738257683814, + "clip_ratio/low_mean": 0.007880916586145759, + "clip_ratio/low_min": 0.007880916586145759, + "clip_ratio/region_mean": 0.01081119041191414, + "completions/clipped_ratio": 0.1953125, + "completions/max_length": 51.0, + "completions/mean_length": 16.0546875, + "completions/min_length": 5.0, + "epoch": 1.587257617728532, + "grad_norm": 2.558849178134003, + "kl": 3.834877057670383, + "learning_rate": 2.222762763569862e-05, + "loss": 0.029992224648594856, + "memory(GiB)": 79.43, + "reward": 0.1875, + "reward_std": 0.35771697759628296, + "rewards/Response_no_think_reward_1/mean": 0.1875, + "rewards/Response_no_think_reward_1/std": 0.8010819554328918, + "step": 143, + "train_speed(iter/s)": 0.007139 + }, + { + "clip_ratio/high_max": 0.003368975827470422, + "clip_ratio/high_mean": 0.003368975827470422, + "clip_ratio/low_mean": 0.02965959811990615, + "clip_ratio/low_min": 0.02965959811990615, + "clip_ratio/region_mean": 0.033028574180207215, + "epoch": 1.5983379501385042, + "grad_norm": 1.5351560901882282, + "kl": 4.278483287169365, + "learning_rate": 2.1085949060360654e-05, + "loss": 0.022547291591763496, + "memory(GiB)": 79.43, + "step": 144, + "train_speed(iter/s)": 0.007121 + }, + { + "clip_ratio/high_max": 0.002464834105921909, + "clip_ratio/high_mean": 0.002464834105921909, + "clip_ratio/low_mean": 0.01630687521537766, + "clip_ratio/low_min": 0.01630687521537766, + "clip_ratio/region_mean": 0.018771709233988076, + "completions/clipped_ratio": 0.1953125, + "completions/max_length": 51.0, + "completions/mean_length": 15.2265625, + "completions/min_length": 6.0, + "epoch": 1.6094182825484764, + "grad_norm": 2.2342359741828877, + "kl": 5.610230388585478, + "learning_rate": 1.9970905297711606e-05, + "loss": 0.04032519459724426, + "memory(GiB)": 79.43, + "reward": 0.15625, + "reward_std": 0.3392276465892792, + "rewards/Response_no_think_reward_1/mean": 0.15625, + "rewards/Response_no_think_reward_1/std": 0.8175004720687866, + "step": 145, + "train_speed(iter/s)": 0.006942 + }, + { + "clip_ratio/high_max": 0.004107788117835298, + "clip_ratio/high_mean": 0.004107788117835298, + "clip_ratio/low_mean": 0.07009978080168366, + "clip_ratio/low_min": 0.07009978080168366, + "clip_ratio/region_mean": 0.07420756877399981, + "epoch": 1.6204986149584486, + "grad_norm": 1.7237867651621368, + "kl": 6.150614712154493, + "learning_rate": 1.888287269367979e-05, + "loss": 0.03451818227767944, + "memory(GiB)": 79.43, + "step": 146, + "train_speed(iter/s)": 0.006925 + }, + { + "clip_ratio/high_max": 0.0020879992516711354, + "clip_ratio/high_mean": 0.0020879992516711354, + "clip_ratio/low_mean": 0.0016714749799575657, + "clip_ratio/low_min": 0.0016714749799575657, + "clip_ratio/region_mean": 0.0037594741152133793, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 51.0, + "completions/mean_length": 12.09375, + "completions/min_length": 5.0, + "epoch": 1.631578947368421, + "grad_norm": 1.8301432866457292, + "kl": 1.3922554631717503, + "learning_rate": 1.7822218477475494e-05, + "loss": 0.015794314444065094, + "memory(GiB)": 79.43, + "reward": 0.5234375, + "reward_std": 0.20214977860450745, + "rewards/Response_no_think_reward_1/mean": 0.5234375, + "rewards/Response_no_think_reward_1/std": 0.6143282055854797, + "step": 147, + "train_speed(iter/s)": 0.006704 + }, + { + "clip_ratio/high_max": 0.004581842658808455, + "clip_ratio/high_mean": 0.004581842658808455, + "clip_ratio/low_mean": 0.00751026114448905, + "clip_ratio/low_min": 0.00751026114448905, + "clip_ratio/region_mean": 0.012092103715986013, + "epoch": 1.6426592797783934, + "grad_norm": 2.0798983048360977, + "kl": 1.5519673400558531, + "learning_rate": 1.6789300637645e-05, + "loss": 0.010683290660381317, + "memory(GiB)": 79.43, + "step": 148, + "train_speed(iter/s)": 0.006691 + }, + { + "clip_ratio/high_max": 0.0013595109921880066, + "clip_ratio/high_mean": 0.0013595109921880066, + "clip_ratio/low_mean": 0.004456432332517579, + "clip_ratio/low_min": 0.004456432332517579, + "clip_ratio/region_mean": 0.005815943295601755, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 51.0, + "completions/mean_length": 13.4453125, + "completions/min_length": 6.0, + "epoch": 1.6537396121883656, + "grad_norm": 2.124340745472962, + "kl": 4.468803564086556, + "learning_rate": 1.578446780124344e-05, + "loss": 0.04191342741250992, + "memory(GiB)": 79.43, + "reward": 0.09375, + "reward_std": 0.2688094973564148, + "rewards/Response_no_think_reward_1/mean": 0.09375, + "rewards/Response_no_think_reward_1/std": 0.7036183476448059, + "step": 149, + "train_speed(iter/s)": 0.006516 + }, + { + "clip_ratio/high_max": 0.004650130198569968, + "clip_ratio/high_mean": 0.004650130198569968, + "clip_ratio/low_mean": 0.015789811441209167, + "clip_ratio/low_min": 0.015789811441209167, + "clip_ratio/region_mean": 0.020439941552467644, + "epoch": 1.6648199445983378, + "grad_norm": 1.3890201997929057, + "kl": 4.818290303461254, + "learning_rate": 1.4808059116167305e-05, + "loss": 0.03111656755208969, + "memory(GiB)": 79.43, + "step": 150, + "train_speed(iter/s)": 0.006505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0031864915508776903, + "clip_ratio/low_min": 0.0031864915508776903, + "clip_ratio/region_mean": 0.0031864915508776903, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 51.0, + "completions/mean_length": 15.96875, + "completions/min_length": 3.0, + "epoch": 1.6759002770083102, + "grad_norm": 1.3263437432774172, + "kl": 3.4037277391953467, + "learning_rate": 1.3860404136686411e-05, + "loss": 0.02711915224790573, + "memory(GiB)": 79.43, + "reward": 0.2890625, + "reward_std": 0.20480823516845703, + "rewards/Response_no_think_reward_1/mean": 0.2890625, + "rewards/Response_no_think_reward_1/std": 0.7852212190628052, + "step": 151, + "train_speed(iter/s)": 0.006367 + }, + { + "clip_ratio/high_max": 0.002190002123825252, + "clip_ratio/high_mean": 0.002190002123825252, + "clip_ratio/low_mean": 0.010535595705732703, + "clip_ratio/low_min": 0.010535595705732703, + "clip_ratio/region_mean": 0.012725598004180938, + "epoch": 1.6869806094182827, + "grad_norm": 1.0936459304561021, + "kl": 3.5067162624327466, + "learning_rate": 1.294182271221377e-05, + "loss": 0.02425944060087204, + "memory(GiB)": 79.43, + "step": 152, + "train_speed(iter/s)": 0.006357 + }, + { + "clip_ratio/high_max": 0.004777037829626352, + "clip_ratio/high_mean": 0.004777037829626352, + "clip_ratio/low_mean": 0.0015179030015133321, + "clip_ratio/low_min": 0.0015179030015133321, + "clip_ratio/region_mean": 0.006294940831139684, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 47.0, + "completions/mean_length": 13.6796875, + "completions/min_length": 3.0, + "epoch": 1.6980609418282548, + "grad_norm": 3.294412855573239, + "kl": 4.274040638643783, + "learning_rate": 1.2052624879351104e-05, + "loss": 0.03787969425320625, + "memory(GiB)": 79.43, + "reward": 0.6015625, + "reward_std": 0.17859892547130585, + "rewards/Response_no_think_reward_1/mean": 0.6015625, + "rewards/Response_no_think_reward_1/std": 0.5931345224380493, + "step": 153, + "train_speed(iter/s)": 0.006206 + }, + { + "clip_ratio/high_max": 0.005947966914391145, + "clip_ratio/high_mean": 0.005947966914391145, + "clip_ratio/low_mean": 0.004764656303450465, + "clip_ratio/low_min": 0.004764656303450465, + "clip_ratio/region_mean": 0.010712623450672254, + "epoch": 1.709141274238227, + "grad_norm": 2.9610057087874977, + "kl": 3.866908519703429, + "learning_rate": 1.119311075724625e-05, + "loss": 0.033227190375328064, + "memory(GiB)": 79.43, + "step": 154, + "train_speed(iter/s)": 0.006199 + }, + { + "clip_ratio/high_max": 0.0032051282469183207, + "clip_ratio/high_mean": 0.0032051282469183207, + "clip_ratio/low_mean": 0.00683464459143579, + "clip_ratio/low_min": 0.00683464459143579, + "clip_ratio/region_mean": 0.01003977283835411, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 51.0, + "completions/mean_length": 16.2265625, + "completions/min_length": 5.0, + "epoch": 1.7202216066481995, + "grad_norm": 3.0112138436872624, + "kl": 4.548647504067048, + "learning_rate": 1.0363570446297999e-05, + "loss": 0.03970736265182495, + "memory(GiB)": 79.43, + "reward": 0.328125, + "reward_std": 0.32412126660346985, + "rewards/Response_no_think_reward_1/mean": 0.328125, + "rewards/Response_no_think_reward_1/std": 0.743574857711792, + "step": 155, + "train_speed(iter/s)": 0.006071 + }, + { + "clip_ratio/high_max": 0.0029493323527276516, + "clip_ratio/high_mean": 0.0029493323527276516, + "clip_ratio/low_mean": 0.008986421715235338, + "clip_ratio/low_min": 0.008986421715235338, + "clip_ratio/region_mean": 0.011935754009755328, + "epoch": 1.7313019390581719, + "grad_norm": 3.1784321577702666, + "kl": 4.521617598744342, + "learning_rate": 9.564283930242257e-06, + "loss": 0.03974776715040207, + "memory(GiB)": 79.43, + "step": 156, + "train_speed(iter/s)": 0.006064 + }, + { + "clip_ratio/high_max": 0.005233740259427577, + "clip_ratio/high_mean": 0.005233740259427577, + "clip_ratio/low_mean": 0.013618246564874426, + "clip_ratio/low_min": 0.013618246564874426, + "clip_ratio/region_mean": 0.018851986795198172, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 46.0, + "completions/mean_length": 12.71875, + "completions/min_length": 2.0, + "epoch": 1.742382271468144, + "grad_norm": 3.107280979259554, + "kl": 7.697673750575632, + "learning_rate": 8.795520981652961e-06, + "loss": 0.04705927148461342, + "memory(GiB)": 79.43, + "reward": 0.171875, + "reward_std": 0.49000412225723267, + "rewards/Response_no_think_reward_1/mean": 0.171875, + "rewards/Response_no_think_reward_1/std": 0.8143339157104492, + "step": 157, + "train_speed(iter/s)": 0.005969 + }, + { + "clip_ratio/high_max": 0.018029920000117272, + "clip_ratio/high_mean": 0.018029920000117272, + "clip_ratio/low_mean": 0.011481558292871341, + "clip_ratio/low_min": 0.011481558292871341, + "clip_ratio/region_mean": 0.029511478263884783, + "epoch": 1.7534626038781163, + "grad_norm": 2.4164221842161395, + "kl": 7.264460427570157, + "learning_rate": 8.05754107088923e-06, + "loss": 0.0432392843067646, + "memory(GiB)": 79.43, + "step": 158, + "train_speed(iter/s)": 0.005963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.003813613730017096, + "clip_ratio/low_min": 0.003813613730017096, + "clip_ratio/region_mean": 0.003813613730017096, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 51.0, + "completions/mean_length": 14.8359375, + "completions/min_length": 4.0, + "epoch": 1.7645429362880887, + "grad_norm": 1.905247433106981, + "kl": 1.1771136652678251, + "learning_rate": 7.350593278519824e-06, + "loss": 0.00986653845757246, + "memory(GiB)": 79.43, + "reward": 0.359375, + "reward_std": 0.27169257402420044, + "rewards/Response_no_think_reward_1/mean": 0.359375, + "rewards/Response_no_think_reward_1/std": 0.7605879902839661, + "step": 159, + "train_speed(iter/s)": 0.005878 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.003413711878238246, + "clip_ratio/low_min": 0.003413711878238246, + "clip_ratio/region_mean": 0.003413711878238246, + "epoch": 1.775623268698061, + "grad_norm": 1.9755946085728415, + "kl": 1.1371133443899453, + "learning_rate": 6.674916211254289e-06, + "loss": 0.009400583803653717, + "memory(GiB)": 79.43, + "step": 160, + "train_speed(iter/s)": 0.005873 + }, + { + "clip_ratio/high_max": 0.0028935185400769114, + "clip_ratio/high_mean": 0.0028935185400769114, + "clip_ratio/low_mean": 0.007061597076244652, + "clip_ratio/low_min": 0.007061597076244652, + "clip_ratio/region_mean": 0.009955115616321564, + "completions/clipped_ratio": 0.1015625, + "completions/max_length": 50.0, + "completions/mean_length": 10.5625, + "completions/min_length": 2.0, + "epoch": 1.7867036011080333, + "grad_norm": 2.0459547550897166, + "kl": 3.880769203417003, + "learning_rate": 7.689418917193289e-05, + "loss": 0.024239888414740562, + "memory(GiB)": 77.36, + "reward": 0.234375, + "reward_std": 0.31300365924835205, + "rewards/Response_no_think_reward_1/mean": 0.234375, + "rewards/Response_no_think_reward_1/std": 0.7475352883338928, + "step": 161, + "train_speed(iter/s)": 0.21784 + }, + { + "clip_ratio/high_max": 0.005787037080153823, + "clip_ratio/high_mean": 0.005787037080153823, + "clip_ratio/low_mean": 0.011741390626411885, + "clip_ratio/low_min": 0.011741390626411885, + "clip_ratio/region_mean": 0.017528427706565708, + "epoch": 1.7977839335180055, + "grad_norm": 1.6284449438133224, + "kl": 3.9539552154019475, + "learning_rate": 7.570198200967362e-05, + "loss": 0.026077013462781906, + "memory(GiB)": 77.36, + "step": 162, + "train_speed(iter/s)": 0.173972 + }, + { + "clip_ratio/high_max": 0.0007267441833391786, + "clip_ratio/high_mean": 0.0007267441833391786, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0007267441833391786, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 51.0, + "completions/mean_length": 15.1875, + "completions/min_length": 6.0, + "epoch": 1.8088642659279779, + "grad_norm": 6.190200790245519, + "kl": 2.1185005172155797, + "learning_rate": 7.451343403954856e-05, + "loss": 0.015710504725575447, + "memory(GiB)": 77.47, + "reward": 0.46875, + "reward_std": 0.11279275268316269, + "rewards/Response_no_think_reward_1/mean": 0.46875, + "rewards/Response_no_think_reward_1/std": 0.7310600280761719, + "step": 163, + "train_speed(iter/s)": 0.105292 + }, + { + "clip_ratio/high_max": 0.010034247010480613, + "clip_ratio/high_mean": 0.010034247010480613, + "clip_ratio/low_mean": 0.0050670221680775285, + "clip_ratio/low_min": 0.0050670221680775285, + "clip_ratio/region_mean": 0.015101269178558141, + "epoch": 1.8199445983379503, + "grad_norm": 1.5020471792427759, + "kl": 1.848701277282089, + "learning_rate": 7.332872425251018e-05, + "loss": 0.011468175798654556, + "memory(GiB)": 77.47, + "step": 164, + "train_speed(iter/s)": 0.094243 + }, + { + "clip_ratio/high_max": 0.0031305171723943204, + "clip_ratio/high_mean": 0.0031305171723943204, + "clip_ratio/low_mean": 0.006101353617850691, + "clip_ratio/low_min": 0.006101353617850691, + "clip_ratio/region_mean": 0.00923187073203735, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 51.0, + "completions/mean_length": 13.578125, + "completions/min_length": 4.0, + "epoch": 1.8310249307479225, + "grad_norm": 1.5859082442887886, + "kl": 3.1664798953570426, + "learning_rate": 7.21480310614947e-05, + "loss": 0.024385500699281693, + "memory(GiB)": 77.47, + "reward": 0.34375, + "reward_std": 0.3683975338935852, + "rewards/Response_no_think_reward_1/mean": 0.34375, + "rewards/Response_no_think_reward_1/std": 0.692337155342102, + "step": 165, + "train_speed(iter/s)": 0.069824 + }, + { + "clip_ratio/high_max": 0.019868265371769667, + "clip_ratio/high_mean": 0.019868265371769667, + "clip_ratio/low_mean": 0.011233113938942552, + "clip_ratio/low_min": 0.011233113938942552, + "clip_ratio/region_mean": 0.031101379543542862, + "epoch": 1.8421052631578947, + "grad_norm": 1.8023731350310037, + "kl": 2.8112484337762, + "learning_rate": 7.097153227455379e-05, + "loss": 0.013743140734732151, + "memory(GiB)": 79.43, + "step": 166, + "train_speed(iter/s)": 0.064907 + }, + { + "clip_ratio/high_max": 0.0011916428920812905, + "clip_ratio/high_mean": 0.0011916428920812905, + "clip_ratio/low_mean": 0.0034684768179431558, + "clip_ratio/low_min": 0.0034684768179431558, + "clip_ratio/region_mean": 0.004660119710024446, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 31.0, + "completions/mean_length": 9.9765625, + "completions/min_length": 3.0, + "epoch": 1.8531855955678669, + "grad_norm": 2.973635940137616, + "kl": 2.157054567243904, + "learning_rate": 6.97994050680772e-05, + "loss": 0.017193181440234184, + "memory(GiB)": 79.43, + "reward": 0.484375, + "reward_std": 0.22119547426700592, + "rewards/Response_no_think_reward_1/mean": 0.484375, + "rewards/Response_no_think_reward_1/std": 0.7527835369110107, + "step": 167, + "train_speed(iter/s)": 0.052203 + }, + { + "clip_ratio/high_max": 0.006835455656982958, + "clip_ratio/high_mean": 0.006835455656982958, + "clip_ratio/low_mean": 0.046273494604974985, + "clip_ratio/low_min": 0.046273494604974985, + "clip_ratio/region_mean": 0.05310894921422005, + "epoch": 1.8642659279778393, + "grad_norm": 1.4603345730407995, + "kl": 3.1518404465168715, + "learning_rate": 6.863182596011087e-05, + "loss": 0.009599600918591022, + "memory(GiB)": 79.43, + "step": 168, + "train_speed(iter/s)": 0.04954 + }, + { + "clip_ratio/high_max": 0.0006965868233237416, + "clip_ratio/high_mean": 0.0006965868233237416, + "clip_ratio/low_mean": 0.002629449882078916, + "clip_ratio/low_min": 0.002629449882078916, + "clip_ratio/region_mean": 0.003326036676298827, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 51.0, + "completions/mean_length": 13.03125, + "completions/min_length": 3.0, + "epoch": 1.8753462603878117, + "grad_norm": 1.302354063269936, + "kl": 1.0980142642074497, + "learning_rate": 6.746897078377372e-05, + "loss": 0.01619286648929119, + "memory(GiB)": 79.43, + "reward": 0.359375, + "reward_std": 0.31949180364608765, + "rewards/Response_no_think_reward_1/mean": 0.359375, + "rewards/Response_no_think_reward_1/std": 0.7810186743736267, + "step": 169, + "train_speed(iter/s)": 0.041791 + }, + { + "clip_ratio/high_max": 0.0037332845386117697, + "clip_ratio/high_mean": 0.0037332845386117697, + "clip_ratio/low_mean": 0.013076976756565273, + "clip_ratio/low_min": 0.013076976756565273, + "clip_ratio/region_mean": 0.016810261004138738, + "epoch": 1.886426592797784, + "grad_norm": 0.6775107653086064, + "kl": 0.8065175649971934, + "learning_rate": 6.6311014660778e-05, + "loss": 0.005674303974956274, + "memory(GiB)": 79.43, + "step": 170, + "train_speed(iter/s)": 0.040129 + }, + { + "clip_ratio/high_max": 0.0017115735390689224, + "clip_ratio/high_mean": 0.0017115735390689224, + "clip_ratio/low_mean": 0.004471359890885651, + "clip_ratio/low_min": 0.004471359890885651, + "clip_ratio/region_mean": 0.006182933488162234, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 51.0, + "completions/mean_length": 10.8359375, + "completions/min_length": 3.0, + "epoch": 1.897506925207756, + "grad_norm": 1.6323523762740895, + "kl": 1.2415385083295405, + "learning_rate": 6.515813197505656e-05, + "loss": 0.01243288628757, + "memory(GiB)": 79.43, + "reward": 0.265625, + "reward_std": 0.25462424755096436, + "rewards/Response_no_think_reward_1/mean": 0.265625, + "rewards/Response_no_think_reward_1/std": 0.7475352883338928, + "step": 171, + "train_speed(iter/s)": 0.035141 + }, + { + "clip_ratio/high_max": 0.015582295309286565, + "clip_ratio/high_mean": 0.015582295309286565, + "clip_ratio/low_mean": 0.010968676884658635, + "clip_ratio/low_min": 0.010968676884658635, + "clip_ratio/region_mean": 0.026550972019322217, + "epoch": 1.9085872576177285, + "grad_norm": 0.8010680473291589, + "kl": 1.0659651298337849, + "learning_rate": 6.401049634650118e-05, + "loss": 4.9868394853547215e-05, + "memory(GiB)": 79.43, + "step": 172, + "train_speed(iter/s)": 0.034005 + }, + { + "clip_ratio/high_max": 0.0024205285299103707, + "clip_ratio/high_mean": 0.0024205285299103707, + "clip_ratio/low_mean": 0.006205722645972855, + "clip_ratio/low_min": 0.006205722645972855, + "clip_ratio/region_mean": 0.008626251190435141, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 51.0, + "completions/mean_length": 10.5, + "completions/min_length": 2.0, + "epoch": 1.919667590027701, + "grad_norm": 2.2598530338882874, + "kl": 3.693746126416954, + "learning_rate": 6.286828060481626e-05, + "loss": 0.02687670849263668, + "memory(GiB)": 79.43, + "reward": 0.328125, + "reward_std": 0.3272990882396698, + "rewards/Response_no_think_reward_1/mean": 0.328125, + "rewards/Response_no_think_reward_1/std": 0.7220855355262756, + "step": 173, + "train_speed(iter/s)": 0.030698 + }, + { + "clip_ratio/high_max": 0.01826251167221926, + "clip_ratio/high_mean": 0.01826251167221926, + "clip_ratio/low_mean": 0.027282171708066016, + "clip_ratio/low_min": 0.027282171708066016, + "clip_ratio/region_mean": 0.04554468300193548, + "epoch": 1.9307479224376731, + "grad_norm": 2.095627271198394, + "kl": 2.8586582938878564, + "learning_rate": 6.173165676349103e-05, + "loss": 0.017260747030377388, + "memory(GiB)": 79.43, + "step": 174, + "train_speed(iter/s)": 0.029861 + }, + { + "clip_ratio/high_max": 0.001936484724865295, + "clip_ratio/high_mean": 0.001936484724865295, + "clip_ratio/low_mean": 0.006831976061221212, + "clip_ratio/low_min": 0.006831976061221212, + "clip_ratio/region_mean": 0.008768460800638422, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 51.0, + "completions/mean_length": 15.4453125, + "completions/min_length": 4.0, + "epoch": 1.9418282548476453, + "grad_norm": 3.0827807424330964, + "kl": 1.9612275282852352, + "learning_rate": 6.060079599389521e-05, + "loss": 0.020160475745797157, + "memory(GiB)": 79.43, + "reward": 0.3125, + "reward_std": 0.29061752557754517, + "rewards/Response_no_think_reward_1/mean": 0.3125, + "rewards/Response_no_think_reward_1/std": 0.6728714108467102, + "step": 175, + "train_speed(iter/s)": 0.027244 + }, + { + "clip_ratio/high_max": 0.002366035128943622, + "clip_ratio/high_mean": 0.002366035128943622, + "clip_ratio/low_mean": 0.02325177617603913, + "clip_ratio/low_min": 0.02325177617603913, + "clip_ratio/region_mean": 0.02561781130498275, + "epoch": 1.9529085872576177, + "grad_norm": 1.4280426957771801, + "kl": 2.292961670376826, + "learning_rate": 5.947586859950103e-05, + "loss": 0.010125662200152874, + "memory(GiB)": 79.43, + "step": 176, + "train_speed(iter/s)": 0.026595 + }, + { + "clip_ratio/high_max": 0.0025192692410200834, + "clip_ratio/high_mean": 0.0025192692410200834, + "clip_ratio/low_mean": 0.008224956574849784, + "clip_ratio/low_min": 0.008224956574849784, + "clip_ratio/region_mean": 0.01074422593228519, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 51.0, + "completions/mean_length": 14.640625, + "completions/min_length": 2.0, + "epoch": 1.9639889196675901, + "grad_norm": 1.6671459210295219, + "kl": 2.1587753768544644, + "learning_rate": 5.83570439902363e-05, + "loss": 0.01727483607828617, + "memory(GiB)": 79.43, + "reward": 0.3515625, + "reward_std": 0.40715324878692627, + "rewards/Response_no_think_reward_1/mean": 0.3515625, + "rewards/Response_no_think_reward_1/std": 0.7273059487342834, + "step": 177, + "train_speed(iter/s)": 0.024303 + }, + { + "clip_ratio/high_max": 0.010156267147976905, + "clip_ratio/high_mean": 0.010156267147976905, + "clip_ratio/low_mean": 0.044217503629624844, + "clip_ratio/low_min": 0.044217503629624844, + "clip_ratio/region_mean": 0.05437377095222473, + "epoch": 1.9750692520775623, + "grad_norm": 1.0768089968910826, + "kl": 2.5309184610377997, + "learning_rate": 5.7244490656971815e-05, + "loss": 0.004669432528316975, + "memory(GiB)": 79.43, + "step": 178, + "train_speed(iter/s)": 0.02381 + }, + { + "clip_ratio/high_max": 0.004732780216727406, + "clip_ratio/high_mean": 0.004732780216727406, + "clip_ratio/low_mean": 0.008979557853308506, + "clip_ratio/low_min": 0.008979557853308506, + "clip_ratio/region_mean": 0.013712337939068675, + "completions/clipped_ratio": 0.171875, + "completions/max_length": 51.0, + "completions/mean_length": 10.34375, + "completions/min_length": 3.0, + "epoch": 1.9861495844875345, + "grad_norm": 5.022277357841641, + "kl": 8.668162640184164, + "learning_rate": 5.613837614614727e-05, + "loss": 0.07592084258794785, + "memory(GiB)": 79.43, + "reward": 0.359375, + "reward_std": 0.43547919392585754, + "rewards/Response_no_think_reward_1/mean": 0.359375, + "rewards/Response_no_think_reward_1/std": 0.7395931482315063, + "step": 179, + "train_speed(iter/s)": 0.021893 + }, + { + "clip_ratio/high_max": 0.012752994283800945, + "clip_ratio/high_mean": 0.012752994283800945, + "clip_ratio/low_mean": 0.013070159242488444, + "clip_ratio/low_min": 0.013070159242488444, + "clip_ratio/region_mean": 0.025823153846431524, + "epoch": 1.997229916897507, + "grad_norm": 3.8459944315903036, + "kl": 5.999185686931014, + "learning_rate": 5.503886703453933e-05, + "loss": 0.05208824574947357, + "memory(GiB)": 79.43, + "step": 180, + "train_speed(iter/s)": 0.02151 + }, + { + "clip_ratio/high_max": 0.003978091655881144, + "clip_ratio/high_mean": 0.003978091655881144, + "clip_ratio/low_mean": 0.010502228324185126, + "clip_ratio/low_min": 0.010502228324185126, + "clip_ratio/region_mean": 0.0144803200091701, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 51.0, + "completions/mean_length": 18.1328125, + "completions/min_length": 3.0, + "epoch": 2.011080332409972, + "grad_norm": 2.078473476427749, + "kl": 4.435361886702594, + "learning_rate": 5.3946128904176e-05, + "loss": 0.04282068833708763, + "memory(GiB)": 79.43, + "reward": 0.3828125, + "reward_std": 0.39536070823669434, + "rewards/Response_no_think_reward_1/mean": 0.3828125, + "rewards/Response_no_think_reward_1/std": 0.7542122602462769, + "step": 181, + "train_speed(iter/s)": 0.020048 + }, + { + "clip_ratio/high_max": 0.008797917238553055, + "clip_ratio/high_mean": 0.008797917238553055, + "clip_ratio/low_mean": 0.032597673358395696, + "clip_ratio/low_min": 0.032597673358395696, + "clip_ratio/region_mean": 0.041395590524189174, + "epoch": 2.0221606648199444, + "grad_norm": 2.4243106079822865, + "kl": 4.070879372608033, + "learning_rate": 5.286032631740023e-05, + "loss": 0.031270649284124374, + "memory(GiB)": 79.43, + "step": 182, + "train_speed(iter/s)": 0.019738 + }, + { + "clip_ratio/high_max": 0.0034528043179307133, + "clip_ratio/high_mean": 0.0034528043179307133, + "clip_ratio/low_mean": 0.0063797495095059276, + "clip_ratio/low_min": 0.0063797495095059276, + "clip_ratio/region_mean": 0.00983255379833281, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 41.0, + "completions/mean_length": 11.1171875, + "completions/min_length": 2.0, + "epoch": 2.033240997229917, + "grad_norm": 2.23823245032825, + "kl": 6.291976309614256, + "learning_rate": 5.1781622792087735e-05, + "loss": 0.035929106175899506, + "memory(GiB)": 79.43, + "reward": 0.3515625, + "reward_std": 0.25183364748954773, + "rewards/Response_no_think_reward_1/mean": 0.3515625, + "rewards/Response_no_think_reward_1/std": 0.7895961403846741, + "step": 183, + "train_speed(iter/s)": 0.018597 + }, + { + "clip_ratio/high_max": 0.005401399568654597, + "clip_ratio/high_mean": 0.005401399568654597, + "clip_ratio/low_mean": 0.017693198169581592, + "clip_ratio/low_min": 0.017693198169581592, + "clip_ratio/region_mean": 0.023094597971066833, + "epoch": 2.044321329639889, + "grad_norm": 1.8820731345647987, + "kl": 5.607178944861516, + "learning_rate": 5.071018077702161e-05, + "loss": 0.026268446817994118, + "memory(GiB)": 79.43, + "step": 184, + "train_speed(iter/s)": 0.018339 + }, + { + "clip_ratio/high_max": 0.0011341095669195056, + "clip_ratio/high_mean": 0.0011341095669195056, + "clip_ratio/low_mean": 0.002640133607201278, + "clip_ratio/low_min": 0.002640133607201278, + "clip_ratio/region_mean": 0.003774243174120784, + "completions/clipped_ratio": 0.1171875, + "completions/max_length": 51.0, + "completions/mean_length": 12.90625, + "completions/min_length": 4.0, + "epoch": 2.0554016620498614, + "grad_norm": 1.162828426123081, + "kl": 2.032533585326746, + "learning_rate": 4.964616162742826e-05, + "loss": 0.018750667572021484, + "memory(GiB)": 79.43, + "reward": 0.4765625, + "reward_std": 0.2650260925292969, + "rewards/Response_no_think_reward_1/mean": 0.4765625, + "rewards/Response_no_think_reward_1/std": 0.7205077409744263, + "step": 185, + "train_speed(iter/s)": 0.017419 + }, + { + "clip_ratio/high_max": 0.023568872129544616, + "clip_ratio/high_mean": 0.023568872129544616, + "clip_ratio/low_mean": 0.003940592927392572, + "clip_ratio/low_min": 0.003940592927392572, + "clip_ratio/region_mean": 0.02750946453306824, + "epoch": 2.0664819944598336, + "grad_norm": 1.5607380960763384, + "kl": 1.8544287201948464, + "learning_rate": 4.8589725580677835e-05, + "loss": 0.017856435850262642, + "memory(GiB)": 79.43, + "step": 186, + "train_speed(iter/s)": 0.017196 + }, + { + "clip_ratio/high_max": 0.0021306818234734237, + "clip_ratio/high_mean": 0.0021306818234734237, + "clip_ratio/low_mean": 0.009969204489607364, + "clip_ratio/low_min": 0.009969204489607364, + "clip_ratio/region_mean": 0.01209988642949611, + "completions/clipped_ratio": 0.1015625, + "completions/max_length": 47.0, + "completions/mean_length": 10.5859375, + "completions/min_length": 3.0, + "epoch": 2.0775623268698062, + "grad_norm": 2.2199233392807804, + "kl": 4.046690948307514, + "learning_rate": 4.754103173215313e-05, + "loss": 0.02398597076535225, + "memory(GiB)": 79.43, + "reward": 0.6171875, + "reward_std": 0.23733052611351013, + "rewards/Response_no_think_reward_1/mean": 0.6171875, + "rewards/Response_no_think_reward_1/std": 0.6413702368736267, + "step": 187, + "train_speed(iter/s)": 0.016352 + }, + { + "clip_ratio/high_max": 0.012073863414116204, + "clip_ratio/high_mean": 0.012073863414116204, + "clip_ratio/low_mean": 0.029048353899270296, + "clip_ratio/low_min": 0.029048353899270296, + "clip_ratio/region_mean": 0.041122217662632465, + "epoch": 2.0886426592797784, + "grad_norm": 0.9728732612498153, + "kl": 4.168077672366053, + "learning_rate": 4.6500238011290295e-05, + "loss": 0.013498052954673767, + "memory(GiB)": 79.43, + "step": 188, + "train_speed(iter/s)": 0.016166 + }, + { + "clip_ratio/high_max": 0.0011479975655674934, + "clip_ratio/high_mean": 0.0011479975655674934, + "clip_ratio/low_mean": 0.01187260658480227, + "clip_ratio/low_min": 0.01187260658480227, + "clip_ratio/region_mean": 0.013020604150369763, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 51.0, + "completions/mean_length": 13.515625, + "completions/min_length": 2.0, + "epoch": 2.0997229916897506, + "grad_norm": 2.9644587346763975, + "kl": 2.9032904393970966, + "learning_rate": 4.546750115779538e-05, + "loss": 0.025254033505916595, + "memory(GiB)": 79.43, + "reward": 0.390625, + "reward_std": 0.3936568796634674, + "rewards/Response_no_think_reward_1/mean": 0.390625, + "rewards/Response_no_think_reward_1/std": 0.815541684627533, + "step": 189, + "train_speed(iter/s)": 0.015408 + }, + { + "clip_ratio/high_max": 0.002348073641769588, + "clip_ratio/high_mean": 0.002348073641769588, + "clip_ratio/low_mean": 0.03768142650369555, + "clip_ratio/low_min": 0.03768142650369555, + "clip_ratio/region_mean": 0.04002950026188046, + "epoch": 2.110803324099723, + "grad_norm": 1.6848916127813878, + "kl": 3.357306843623519, + "learning_rate": 4.444297669803981e-05, + "loss": 0.011942790821194649, + "memory(GiB)": 79.43, + "step": 190, + "train_speed(iter/s)": 0.01525 + }, + { + "clip_ratio/high_max": 0.0038575184298679233, + "clip_ratio/high_mean": 0.0038575184298679233, + "clip_ratio/low_mean": 0.0018564801721367985, + "clip_ratio/low_min": 0.0018564801721367985, + "clip_ratio/region_mean": 0.005713998660212383, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 49.0, + "completions/mean_length": 12.40625, + "completions/min_length": 2.0, + "epoch": 2.1218836565096955, + "grad_norm": 2.185626617314445, + "kl": 4.500719710253179, + "learning_rate": 4.342681892163868e-05, + "loss": 0.04343116655945778, + "memory(GiB)": 79.43, + "reward": 0.3125, + "reward_std": 0.3249424993991852, + "rewards/Response_no_think_reward_1/mean": 0.3125, + "rewards/Response_no_think_reward_1/std": 0.8108515739440918, + "step": 191, + "train_speed(iter/s)": 0.014605 + }, + { + "clip_ratio/high_max": 0.005653464584611356, + "clip_ratio/high_mean": 0.005653464584611356, + "clip_ratio/low_mean": 0.014596144377719611, + "clip_ratio/low_min": 0.014596144377719611, + "clip_ratio/region_mean": 0.020249608729500324, + "epoch": 2.1329639889196677, + "grad_norm": 1.3831255673808591, + "kl": 3.3093000794760883, + "learning_rate": 4.241918085821547e-05, + "loss": 0.025410175323486328, + "memory(GiB)": 79.43, + "step": 192, + "train_speed(iter/s)": 0.014469 + }, + { + "clip_ratio/high_max": 0.0007621950935572386, + "clip_ratio/high_mean": 0.0007621950935572386, + "clip_ratio/low_mean": 0.016430354735348374, + "clip_ratio/low_min": 0.016430354735348374, + "clip_ratio/region_mean": 0.017192550061736256, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 52.0, + "completions/mean_length": 13.7578125, + "completions/min_length": 3.0, + "epoch": 2.14404432132964, + "grad_norm": 4.027535318826777, + "kl": 5.601793970912695, + "learning_rate": 4.142021425435612e-05, + "loss": 0.06872062385082245, + "memory(GiB)": 79.43, + "reward": 0.171875, + "reward_std": 0.3266732692718506, + "rewards/Response_no_think_reward_1/mean": 0.171875, + "rewards/Response_no_think_reward_1/std": 0.8334481716156006, + "step": 193, + "train_speed(iter/s)": 0.013947 + }, + { + "clip_ratio/high_max": 0.0032193634542636573, + "clip_ratio/high_mean": 0.0032193634542636573, + "clip_ratio/low_mean": 0.0076680079801008105, + "clip_ratio/low_min": 0.0076680079801008105, + "clip_ratio/region_mean": 0.010887371434364468, + "epoch": 2.155124653739612, + "grad_norm": 4.385106696987604, + "kl": 3.2986946790479124, + "learning_rate": 4.0430069550756665e-05, + "loss": 0.057711075991392136, + "memory(GiB)": 79.43, + "step": 194, + "train_speed(iter/s)": 0.013826 + }, + { + "clip_ratio/high_max": 0.0006905972259119153, + "clip_ratio/high_mean": 0.0006905972259119153, + "clip_ratio/low_mean": 0.0036786437267437577, + "clip_ratio/low_min": 0.0036786437267437577, + "clip_ratio/region_mean": 0.004369240894448012, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 49.0, + "completions/mean_length": 12.3359375, + "completions/min_length": 6.0, + "epoch": 2.1662049861495847, + "grad_norm": 1.606761596776354, + "kl": 1.8307960720267147, + "learning_rate": 3.944889585956746e-05, + "loss": 0.01529858261346817, + "memory(GiB)": 79.43, + "reward": 0.53125, + "reward_std": 0.13994136452674866, + "rewards/Response_no_think_reward_1/mean": 0.53125, + "rewards/Response_no_think_reward_1/std": 0.6010162234306335, + "step": 195, + "train_speed(iter/s)": 0.01337 + }, + { + "clip_ratio/high_max": 0.0014173414674587548, + "clip_ratio/high_mean": 0.0014173414674587548, + "clip_ratio/low_mean": 0.01785696716979146, + "clip_ratio/low_min": 0.01785696716979146, + "clip_ratio/region_mean": 0.019274308579042554, + "epoch": 2.177285318559557, + "grad_norm": 0.6605318527442696, + "kl": 1.4683757405728102, + "learning_rate": 3.847684094193733e-05, + "loss": 0.006466372404247522, + "memory(GiB)": 79.43, + "step": 196, + "train_speed(iter/s)": 0.013262 + }, + { + "clip_ratio/high_max": 0.0005787037080153823, + "clip_ratio/high_mean": 0.0005787037080153823, + "clip_ratio/low_mean": 0.00706394191365689, + "clip_ratio/low_min": 0.00706394191365689, + "clip_ratio/region_mean": 0.007642645505256951, + "completions/clipped_ratio": 0.1171875, + "completions/max_length": 51.0, + "completions/mean_length": 12.6640625, + "completions/min_length": 4.0, + "epoch": 2.188365650969529, + "grad_norm": 2.268325311129086, + "kl": 1.9495291512284894, + "learning_rate": 3.751405118576138e-05, + "loss": 0.021440906450152397, + "memory(GiB)": 79.43, + "reward": 0.40625, + "reward_std": 0.33038538694381714, + "rewards/Response_no_think_reward_1/mean": 0.40625, + "rewards/Response_no_think_reward_1/std": 0.692337155342102, + "step": 197, + "train_speed(iter/s)": 0.012815 + }, + { + "clip_ratio/high_max": 0.0015153939457377419, + "clip_ratio/high_mean": 0.0015153939457377419, + "clip_ratio/low_mean": 0.03491706313798204, + "clip_ratio/low_min": 0.03491706313798204, + "clip_ratio/region_mean": 0.03643245715647936, + "epoch": 2.1994459833795013, + "grad_norm": 1.065092801753265, + "kl": 2.4326230198785197, + "learning_rate": 3.6560671583635467e-05, + "loss": 0.014199762605130672, + "memory(GiB)": 79.43, + "step": 198, + "train_speed(iter/s)": 0.012721 + }, + { + "clip_ratio/high_max": 0.002581158187240362, + "clip_ratio/high_mean": 0.002581158187240362, + "clip_ratio/low_mean": 0.0033617604058235884, + "clip_ratio/low_min": 0.0033617604058235884, + "clip_ratio/region_mean": 0.0059429185930639505, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 51.0, + "completions/mean_length": 11.40625, + "completions/min_length": 2.0, + "epoch": 2.2105263157894735, + "grad_norm": 4.969169060239286, + "kl": 2.6644327271496877, + "learning_rate": 3.561684571102087e-05, + "loss": 0.04970448464155197, + "memory(GiB)": 79.43, + "reward": 0.609375, + "reward_std": 0.189372718334198, + "rewards/Response_no_think_reward_1/mean": 0.609375, + "rewards/Response_no_think_reward_1/std": 0.617773175239563, + "step": 199, + "train_speed(iter/s)": 0.012318 + }, + { + "clip_ratio/high_max": 0.007839692523702979, + "clip_ratio/high_mean": 0.007839692523702979, + "clip_ratio/low_mean": 0.007207914255559444, + "clip_ratio/low_min": 0.007207914255559444, + "clip_ratio/region_mean": 0.015047606895677745, + "epoch": 2.221606648199446, + "grad_norm": 1.5017699084351217, + "kl": 1.7442057720618322, + "learning_rate": 3.468271570462235e-05, + "loss": 0.03236105293035507, + "memory(GiB)": 79.43, + "step": 200, + "train_speed(iter/s)": 0.012235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.011193181620910764, + "clip_ratio/low_min": 0.011193181620910764, + "clip_ratio/region_mean": 0.011193181620910764, + "completions/clipped_ratio": 0.109375, + "completions/max_length": 51.0, + "completions/mean_length": 14.9140625, + "completions/min_length": 3.0, + "epoch": 2.2326869806094183, + "grad_norm": 4.931712878754855, + "kl": 5.507446703501046, + "learning_rate": 3.375842224098281e-05, + "loss": 0.04869800806045532, + "memory(GiB)": 79.43, + "reward": 0.0546875, + "reward_std": 0.13610614836215973, + "rewards/Response_no_think_reward_1/mean": 0.0546875, + "rewards/Response_no_think_reward_1/std": 0.9076108336448669, + "step": 201, + "train_speed(iter/s)": 0.011859 + }, + { + "clip_ratio/high_max": 0.0006249999860301614, + "clip_ratio/high_mean": 0.0006249999860301614, + "clip_ratio/low_mean": 0.01068181823939085, + "clip_ratio/low_min": 0.01068181823939085, + "clip_ratio/region_mean": 0.011306818574666977, + "epoch": 2.2437673130193905, + "grad_norm": 1.976139653305045, + "kl": 2.772067047073506, + "learning_rate": 3.2844104515298155e-05, + "loss": 0.02979501150548458, + "memory(GiB)": 79.43, + "step": 202, + "train_speed(iter/s)": 0.011784 + }, + { + "clip_ratio/high_max": 0.0021551724057644606, + "clip_ratio/high_mean": 0.0021551724057644606, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0021551724057644606, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 51.0, + "completions/mean_length": 15.109375, + "completions/min_length": 2.0, + "epoch": 2.254847645429363, + "grad_norm": 1.5562382605817529, + "kl": 0.8431436920873239, + "learning_rate": 3.19399002204547e-05, + "loss": 0.007927711121737957, + "memory(GiB)": 79.43, + "reward": 0.5546875, + "reward_std": 0.1965562254190445, + "rewards/Response_no_think_reward_1/mean": 0.5546875, + "rewards/Response_no_think_reward_1/std": 0.612322211265564, + "step": 203, + "train_speed(iter/s)": 0.011458 + }, + { + "clip_ratio/high_max": 0.01512533612549305, + "clip_ratio/high_mean": 0.01512533612549305, + "clip_ratio/low_mean": 0.011428028345108032, + "clip_ratio/low_min": 0.011428028345108032, + "clip_ratio/region_mean": 0.026553363888524473, + "epoch": 2.2659279778393353, + "grad_norm": 0.7974321717950155, + "kl": 0.8073496993965819, + "learning_rate": 3.104594552629331e-05, + "loss": -0.0036839484237134457, + "memory(GiB)": 79.43, + "step": 204, + "train_speed(iter/s)": 0.011391 + }, + { + "clip_ratio/high_max": 0.0038362154737114906, + "clip_ratio/high_mean": 0.0038362154737114906, + "clip_ratio/low_mean": 0.006098126468714327, + "clip_ratio/low_min": 0.006098126468714327, + "clip_ratio/region_mean": 0.009934341884218156, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 51.0, + "completions/mean_length": 9.859375, + "completions/min_length": 3.0, + "epoch": 2.2770083102493075, + "grad_norm": 1.6738122176232313, + "kl": 2.2672509452095255, + "learning_rate": 3.016237505910272e-05, + "loss": 0.019055092707276344, + "memory(GiB)": 79.43, + "reward": 0.3828125, + "reward_std": 0.24184317886829376, + "rewards/Response_no_think_reward_1/mean": 0.3828125, + "rewards/Response_no_think_reward_1/std": 0.6289736032485962, + "step": 205, + "train_speed(iter/s)": 0.011117 + }, + { + "clip_ratio/high_max": 0.008882887894287705, + "clip_ratio/high_mean": 0.008882887894287705, + "clip_ratio/low_mean": 0.0127694628899917, + "clip_ratio/low_min": 0.0127694628899917, + "clip_ratio/region_mean": 0.021652350667864084, + "epoch": 2.2880886426592797, + "grad_norm": 1.7770705331591035, + "kl": 2.6307186615886167, + "learning_rate": 2.9289321881345254e-05, + "loss": 0.020140212029218674, + "memory(GiB)": 79.43, + "step": 206, + "train_speed(iter/s)": 0.011054 + }, + { + "clip_ratio/high_max": 0.0027895565144717693, + "clip_ratio/high_mean": 0.0027895565144717693, + "clip_ratio/low_mean": 0.004148252191953361, + "clip_ratio/low_min": 0.004148252191953361, + "clip_ratio/region_mean": 0.0069378085900098085, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 51.0, + "completions/mean_length": 11.28125, + "completions/min_length": 2.0, + "epoch": 2.299168975069252, + "grad_norm": 1.319228923221061, + "kl": 2.2385971848852932, + "learning_rate": 2.8426917471618144e-05, + "loss": 0.02266935259103775, + "memory(GiB)": 79.43, + "reward": 0.3359375, + "reward_std": 0.2283492535352707, + "rewards/Response_no_think_reward_1/mean": 0.3359375, + "rewards/Response_no_think_reward_1/std": 0.7018237709999084, + "step": 207, + "train_speed(iter/s)": 0.010791 + }, + { + "clip_ratio/high_max": 0.011985209857812151, + "clip_ratio/high_mean": 0.011985209857812151, + "clip_ratio/low_mean": 0.007749008131213486, + "clip_ratio/low_min": 0.007749008131213486, + "clip_ratio/region_mean": 0.019734218367375433, + "epoch": 2.3102493074792245, + "grad_norm": 1.4392087091443533, + "kl": 2.1376722818240523, + "learning_rate": 2.7575291704853323e-05, + "loss": 0.019175298511981964, + "memory(GiB)": 79.43, + "step": 208, + "train_speed(iter/s)": 0.010735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.005309505155310035, + "clip_ratio/low_min": 0.005309505155310035, + "clip_ratio/region_mean": 0.005309505155310035, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 51.0, + "completions/mean_length": 13.78125, + "completions/min_length": 2.0, + "epoch": 2.3213296398891967, + "grad_norm": 3.9723735950660957, + "kl": 5.278170272591524, + "learning_rate": 2.673457283275873e-05, + "loss": 0.035180311650037766, + "memory(GiB)": 79.43, + "reward": 0.40625, + "reward_std": 0.13098980486392975, + "rewards/Response_no_think_reward_1/mean": 0.40625, + "rewards/Response_no_think_reward_1/std": 0.6692044734954834, + "step": 209, + "train_speed(iter/s)": 0.010491 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.005584693106357008, + "clip_ratio/low_min": 0.005584693106357008, + "clip_ratio/region_mean": 0.005584693106357008, + "epoch": 2.332409972299169, + "grad_norm": 1.3665368093756183, + "kl": 2.833973544766195, + "learning_rate": 2.5904887464504114e-05, + "loss": 0.021212518215179443, + "memory(GiB)": 79.43, + "step": 210, + "train_speed(iter/s)": 0.010441 + }, + { + "clip_ratio/high_max": 0.0037202382227405906, + "clip_ratio/high_mean": 0.0037202382227405906, + "clip_ratio/low_mean": 0.0047081211232580245, + "clip_ratio/low_min": 0.0047081211232580245, + "clip_ratio/region_mean": 0.008428359229583293, + "completions/clipped_ratio": 0.109375, + "completions/max_length": 51.0, + "completions/mean_length": 13.515625, + "completions/min_length": 2.0, + "epoch": 2.343490304709141, + "grad_norm": 13.071565563919009, + "kl": 3.0507688876241446, + "learning_rate": 2.5086360547654087e-05, + "loss": 0.03715192899107933, + "memory(GiB)": 79.43, + "reward": 0.3515625, + "reward_std": 0.23045912384986877, + "rewards/Response_no_think_reward_1/mean": 0.3515625, + "rewards/Response_no_think_reward_1/std": 0.8189665675163269, + "step": 211, + "train_speed(iter/s)": 0.010202 + }, + { + "clip_ratio/high_max": 0.003481080086203292, + "clip_ratio/high_mean": 0.003481080086203292, + "clip_ratio/low_mean": 0.006368864676915109, + "clip_ratio/low_min": 0.006368864676915109, + "clip_ratio/region_mean": 0.009849944588495418, + "epoch": 2.3545706371191137, + "grad_norm": 3.097490825276586, + "kl": 2.710134948603809, + "learning_rate": 2.4279115349351543e-05, + "loss": 0.022687038406729698, + "memory(GiB)": 79.43, + "step": 212, + "train_speed(iter/s)": 0.010156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.002016128972172737, + "clip_ratio/low_min": 0.002016128972172737, + "clip_ratio/region_mean": 0.002016128972172737, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 51.0, + "completions/mean_length": 10.421875, + "completions/min_length": 2.0, + "epoch": 2.365650969529086, + "grad_norm": 1.136594776957956, + "kl": 1.1303195441141725, + "learning_rate": 2.3483273437754107e-05, + "loss": 0.009659601375460625, + "memory(GiB)": 79.43, + "reward": 0.390625, + "reward_std": 0.10205793380737305, + "rewards/Response_no_think_reward_1/mean": 0.390625, + "rewards/Response_no_think_reward_1/std": 0.7342506647109985, + "step": 213, + "train_speed(iter/s)": 0.009943 + }, + { + "clip_ratio/high_max": 0.006048386916518211, + "clip_ratio/high_mean": 0.006048386916518211, + "clip_ratio/low_mean": 0.00942321156617254, + "clip_ratio/low_min": 0.00942321156617254, + "clip_ratio/region_mean": 0.015471598482690752, + "epoch": 2.376731301939058, + "grad_norm": 0.825911363622962, + "kl": 1.1256521647737827, + "learning_rate": 2.26989546637263e-05, + "loss": 0.005634521599858999, + "memory(GiB)": 79.43, + "step": 214, + "train_speed(iter/s)": 0.009901 + }, + { + "clip_ratio/high_max": 0.001342517207376659, + "clip_ratio/high_mean": 0.001342517207376659, + "clip_ratio/low_mean": 0.0013586956774815917, + "clip_ratio/low_min": 0.0013586956774815917, + "clip_ratio/region_mean": 0.0027012128848582506, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 51.0, + "completions/mean_length": 15.6484375, + "completions/min_length": 4.0, + "epoch": 2.3878116343490303, + "grad_norm": 1.6607743819090053, + "kl": 1.4292651428841054, + "learning_rate": 2.1926277142790552e-05, + "loss": 0.01813752017915249, + "memory(GiB)": 79.43, + "reward": 0.609375, + "reward_std": 0.31577742099761963, + "rewards/Response_no_think_reward_1/mean": 0.609375, + "rewards/Response_no_think_reward_1/std": 0.6668102145195007, + "step": 215, + "train_speed(iter/s)": 0.009685 + }, + { + "clip_ratio/high_max": 0.0026189450873062015, + "clip_ratio/high_mean": 0.0026189450873062015, + "clip_ratio/low_mean": 0.0053210301557555795, + "clip_ratio/low_min": 0.0053210301557555795, + "clip_ratio/region_mean": 0.007939975359477103, + "epoch": 2.398891966759003, + "grad_norm": 1.3712211159888634, + "kl": 1.414744115434587, + "learning_rate": 2.116535723733938e-05, + "loss": 0.01291576400399208, + "memory(GiB)": 79.43, + "step": 216, + "train_speed(iter/s)": 0.009645 + }, + { + "clip_ratio/high_max": 0.002932063478510827, + "clip_ratio/high_mean": 0.002932063478510827, + "clip_ratio/low_mean": 0.0015625000232830644, + "clip_ratio/low_min": 0.0015625000232830644, + "clip_ratio/region_mean": 0.004494563501793891, + "completions/clipped_ratio": 0.0703125, + "completions/max_length": 49.0, + "completions/mean_length": 10.9296875, + "completions/min_length": 2.0, + "epoch": 2.409972299168975, + "grad_norm": 6.046911872213327, + "kl": 3.113054335270135, + "learning_rate": 2.0416309539111654e-05, + "loss": 0.050445497035980225, + "memory(GiB)": 79.43, + "reward": 0.484375, + "reward_std": 0.2109457552433014, + "rewards/Response_no_think_reward_1/mean": 0.484375, + "rewards/Response_no_think_reward_1/std": 0.8416741490364075, + "step": 217, + "train_speed(iter/s)": 0.009417 + }, + { + "clip_ratio/high_max": 0.0037691170582547784, + "clip_ratio/high_mean": 0.0037691170582547784, + "clip_ratio/low_mean": 0.0015625000232830644, + "clip_ratio/low_min": 0.0015625000232830644, + "clip_ratio/region_mean": 0.005331617081537843, + "epoch": 2.4210526315789473, + "grad_norm": 5.501927716874391, + "kl": 2.3150660254395916, + "learning_rate": 1.967924685193552e-05, + "loss": 0.030173499137163162, + "memory(GiB)": 79.43, + "step": 218, + "train_speed(iter/s)": 0.009382 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 47.0, + "completions/mean_length": 7.734375, + "completions/min_length": 2.0, + "epoch": 2.4321329639889195, + "grad_norm": 1.8000862094404895, + "kl": 1.0117176891799318, + "learning_rate": 1.8954280174740537e-05, + "loss": 0.009762465953826904, + "memory(GiB)": 79.43, + "reward": 0.7265625, + "reward_std": 0.19728107750415802, + "rewards/Response_no_think_reward_1/mean": 0.7265625, + "rewards/Response_no_think_reward_1/std": 0.5130554437637329, + "step": 219, + "train_speed(iter/s)": 0.009197 + }, + { + "clip_ratio/high_max": 0.012120938044972718, + "clip_ratio/high_mean": 0.012120938044972718, + "clip_ratio/low_mean": 0.008152684022206813, + "clip_ratio/low_min": 0.008152684022206813, + "clip_ratio/region_mean": 0.020273622183594853, + "epoch": 2.443213296398892, + "grad_norm": 1.1304227715929336, + "kl": 1.0098483412293717, + "learning_rate": 1.824151868484164e-05, + "loss": 0.002491909544914961, + "memory(GiB)": 79.43, + "step": 220, + "train_speed(iter/s)": 0.009165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 38.0, + "completions/mean_length": 10.0546875, + "completions/min_length": 2.0, + "epoch": 2.4542936288088644, + "grad_norm": 0.7487543034262131, + "kl": 2.248408433049917, + "learning_rate": 1.7541069721497493e-05, + "loss": 0.020080924034118652, + "memory(GiB)": 79.43, + "reward": 0.5234375, + "reward_std": 0.0765409916639328, + "rewards/Response_no_think_reward_1/mean": 0.5234375, + "rewards/Response_no_think_reward_1/std": 0.8320815563201904, + "step": 221, + "train_speed(iter/s)": 0.008978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.002016128972172737, + "clip_ratio/low_min": 0.002016128972172737, + "clip_ratio/region_mean": 0.002016128972172737, + "epoch": 2.4653739612188366, + "grad_norm": 0.5471388079740459, + "kl": 2.286591000854969, + "learning_rate": 1.6853038769745467e-05, + "loss": 0.020101318135857582, + "memory(GiB)": 79.43, + "step": 222, + "train_speed(iter/s)": 0.008949 + }, + { + "clip_ratio/high_max": 0.00021186440426390618, + "clip_ratio/high_mean": 0.00021186440426390618, + "clip_ratio/low_mean": 0.008871822035871446, + "clip_ratio/low_min": 0.008871822035871446, + "clip_ratio/region_mean": 0.009083686396479607, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 51.0, + "completions/mean_length": 13.7109375, + "completions/min_length": 2.0, + "epoch": 2.4764542936288088, + "grad_norm": 1.362062642699485, + "kl": 2.5377135479357094, + "learning_rate": 1.6177529444516194e-05, + "loss": 0.02206542156636715, + "memory(GiB)": 79.43, + "reward": 0.296875, + "reward_std": 0.19568344950675964, + "rewards/Response_no_think_reward_1/mean": 0.296875, + "rewards/Response_no_think_reward_1/std": 0.8993381261825562, + "step": 223, + "train_speed(iter/s)": 0.008782 + }, + { + "clip_ratio/high_max": 0.0016772599192336202, + "clip_ratio/high_mean": 0.0016772599192336202, + "clip_ratio/low_mean": 0.006158006319310516, + "clip_ratio/low_min": 0.006158006319310516, + "clip_ratio/region_mean": 0.007835266180336475, + "epoch": 2.487534626038781, + "grad_norm": 1.146370544556826, + "kl": 2.2770726842572913, + "learning_rate": 1.551464347502929e-05, + "loss": 0.019708380103111267, + "memory(GiB)": 79.43, + "step": 224, + "train_speed(iter/s)": 0.008755 + }, + { + "clip_ratio/high_max": 0.0006412798393284902, + "clip_ratio/high_mean": 0.0006412798393284902, + "clip_ratio/low_mean": 0.005606794758932665, + "clip_ratio/low_min": 0.005606794758932665, + "clip_ratio/region_mean": 0.006248074598261155, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 51.0, + "completions/mean_length": 11.546875, + "completions/min_length": 3.0, + "epoch": 2.4986149584487536, + "grad_norm": 15.413339320982411, + "kl": 8.634065281590665, + "learning_rate": 1.486448068947348e-05, + "loss": 0.12368878722190857, + "memory(GiB)": 79.43, + "reward": 0.3828125, + "reward_std": 0.24082913994789124, + "rewards/Response_no_think_reward_1/mean": 0.3828125, + "rewards/Response_no_think_reward_1/std": 0.7000685334205627, + "step": 225, + "train_speed(iter/s)": 0.008606 + }, + { + "clip_ratio/high_max": 0.002233636550954543, + "clip_ratio/high_mean": 0.002233636550954543, + "clip_ratio/low_mean": 0.0067390410986263305, + "clip_ratio/low_min": 0.0067390410986263305, + "clip_ratio/region_mean": 0.008972677649580874, + "epoch": 2.509695290858726, + "grad_norm": 9.203465486887826, + "kl": 6.063290272684753, + "learning_rate": 1.42271389999728e-05, + "loss": 0.08513150364160538, + "memory(GiB)": 79.43, + "step": 226, + "train_speed(iter/s)": 0.008581 + }, + { + "clip_ratio/high_max": 0.0003799392143264413, + "clip_ratio/high_mean": 0.0003799392143264413, + "clip_ratio/low_mean": 0.0011398176429793239, + "clip_ratio/low_min": 0.0011398176429793239, + "clip_ratio/region_mean": 0.0015197568573057652, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 51.0, + "completions/mean_length": 10.0390625, + "completions/min_length": 3.0, + "epoch": 2.520775623268698, + "grad_norm": 2.120226005232822, + "kl": 0.9155051370617002, + "learning_rate": 1.360271438784133e-05, + "loss": 0.010180685669183731, + "memory(GiB)": 79.43, + "reward": 0.1484375, + "reward_std": 0.1979907602071762, + "rewards/Response_no_think_reward_1/mean": 0.1484375, + "rewards/Response_no_think_reward_1/std": 0.7326990962028503, + "step": 227, + "train_speed(iter/s)": 0.008446 + }, + { + "clip_ratio/high_max": 0.0009498480358161032, + "clip_ratio/high_mean": 0.0009498480358161032, + "clip_ratio/low_mean": 0.009764322196133435, + "clip_ratio/low_min": 0.009764322196133435, + "clip_ratio/region_mean": 0.010714170290157199, + "epoch": 2.5318559556786706, + "grad_norm": 1.4078021108161598, + "kl": 0.9818148800404742, + "learning_rate": 1.2991300889128866e-05, + "loss": 0.004585812333971262, + "memory(GiB)": 79.43, + "step": 228, + "train_speed(iter/s)": 0.008423 + }, + { + "clip_ratio/high_max": 0.0011812333250418305, + "clip_ratio/high_mean": 0.0011812333250418305, + "clip_ratio/low_mean": 0.0039037082460708916, + "clip_ratio/low_min": 0.0039037082460708916, + "clip_ratio/region_mean": 0.005084941629320383, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 51.0, + "completions/mean_length": 8.265625, + "completions/min_length": 2.0, + "epoch": 2.542936288088643, + "grad_norm": 5.0825431269611965, + "kl": 1.4881244308780879, + "learning_rate": 1.2392990580459352e-05, + "loss": 0.01816009357571602, + "memory(GiB)": 79.43, + "reward": 0.265625, + "reward_std": 0.19226360321044922, + "rewards/Response_no_think_reward_1/mean": 0.265625, + "rewards/Response_no_think_reward_1/std": 0.9004318714141846, + "step": 229, + "train_speed(iter/s)": 0.008282 + }, + { + "clip_ratio/high_max": 0.0009686482953839004, + "clip_ratio/high_mean": 0.0009686482953839004, + "clip_ratio/low_mean": 0.00444972823606804, + "clip_ratio/low_min": 0.00444972823606804, + "clip_ratio/region_mean": 0.0054183765314519405, + "epoch": 2.554016620498615, + "grad_norm": 5.947762469687869, + "kl": 1.5637177263852209, + "learning_rate": 1.1807873565164506e-05, + "loss": 0.014085257425904274, + "memory(GiB)": 79.43, + "step": 230, + "train_speed(iter/s)": 0.008261 + }, + { + "clip_ratio/high_max": 0.001238734505022876, + "clip_ratio/high_mean": 0.001238734505022876, + "clip_ratio/low_mean": 0.0045786225236952305, + "clip_ratio/low_min": 0.0045786225236952305, + "clip_ratio/region_mean": 0.0058173570287181064, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 51.0, + "completions/mean_length": 15.609375, + "completions/min_length": 3.0, + "epoch": 2.565096952908587, + "grad_norm": 2.991093002864201, + "kl": 2.183058348018676, + "learning_rate": 1.1236037959714618e-05, + "loss": 0.021889833733439445, + "memory(GiB)": 79.43, + "reward": 0.4609375, + "reward_std": 0.3135277330875397, + "rewards/Response_no_think_reward_1/mean": 0.4609375, + "rewards/Response_no_think_reward_1/std": 0.7827102541923523, + "step": 231, + "train_speed(iter/s)": 0.008131 + }, + { + "clip_ratio/high_max": 0.0028425113996490836, + "clip_ratio/high_mean": 0.0028425113996490836, + "clip_ratio/low_mean": 0.009468507007113658, + "clip_ratio/low_min": 0.009468507007113658, + "clip_ratio/region_mean": 0.012311018639593385, + "epoch": 2.5761772853185594, + "grad_norm": 2.565607083550086, + "kl": 1.9908064976334572, + "learning_rate": 1.067756988044848e-05, + "loss": 0.016938552260398865, + "memory(GiB)": 79.43, + "step": 232, + "train_speed(iter/s)": 0.008111 + }, + { + "clip_ratio/high_max": 0.0014164648891892284, + "clip_ratio/high_mean": 0.0014164648891892284, + "clip_ratio/low_mean": 0.0031525760132353753, + "clip_ratio/low_min": 0.0031525760132353753, + "clip_ratio/region_mean": 0.004569040902424604, + "completions/clipped_ratio": 0.1015625, + "completions/max_length": 51.0, + "completions/mean_length": 9.75, + "completions/min_length": 2.0, + "epoch": 2.587257617728532, + "grad_norm": 5.0565474550432326, + "kl": 5.047875659758574, + "learning_rate": 1.0132553430604608e-05, + "loss": 0.07161970436573029, + "memory(GiB)": 79.43, + "reward": 0.2265625, + "reward_std": 0.17859892547130585, + "rewards/Response_no_think_reward_1/mean": 0.2265625, + "rewards/Response_no_think_reward_1/std": 0.7446908354759216, + "step": 233, + "train_speed(iter/s)": 0.008004 + }, + { + "clip_ratio/high_max": 0.006010896002408117, + "clip_ratio/high_mean": 0.006010896002408117, + "clip_ratio/low_mean": 0.006369341921526939, + "clip_ratio/low_min": 0.006369341921526939, + "clip_ratio/region_mean": 0.01238023757468909, + "epoch": 2.598337950138504, + "grad_norm": 3.9632773907657626, + "kl": 4.599423869600287, + "learning_rate": 9.601070687655667e-06, + "loss": 0.062261972576379776, + "memory(GiB)": 79.43, + "step": 234, + "train_speed(iter/s)": 0.007986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.001923076924867928, + "clip_ratio/low_min": 0.001923076924867928, + "clip_ratio/region_mean": 0.001923076924867928, + "completions/clipped_ratio": 0.1015625, + "completions/max_length": 51.0, + "completions/mean_length": 9.453125, + "completions/min_length": 2.0, + "epoch": 2.6094182825484764, + "grad_norm": 5.621375964647713, + "kl": 4.620725775370374, + "learning_rate": 9.083201690947763e-06, + "loss": 0.0421258881688118, + "memory(GiB)": 79.43, + "reward": 0.28125, + "reward_std": 0.1468440443277359, + "rewards/Response_no_think_reward_1/mean": 0.28125, + "rewards/Response_no_think_reward_1/std": 0.7830638289451599, + "step": 235, + "train_speed(iter/s)": 0.007861 + }, + { + "clip_ratio/high_max": 0.010077075916342437, + "clip_ratio/high_mean": 0.010077075916342437, + "clip_ratio/low_mean": 0.000961538462433964, + "clip_ratio/low_min": 0.000961538462433964, + "clip_ratio/region_mean": 0.01103861432056874, + "epoch": 2.6204986149584486, + "grad_norm": 4.170803291082545, + "kl": 4.307271543191746, + "learning_rate": 8.579024429646932e-06, + "loss": 0.03107025846838951, + "memory(GiB)": 79.43, + "step": 236, + "train_speed(iter/s)": 0.007844 + }, + { + "clip_ratio/high_max": 0.00024319066142197698, + "clip_ratio/high_mean": 0.00024319066142197698, + "clip_ratio/low_mean": 0.0016447368543595076, + "clip_ratio/low_min": 0.0016447368543595076, + "clip_ratio/region_mean": 0.0018879275157814845, + "completions/clipped_ratio": 0.1015625, + "completions/max_length": 51.0, + "completions/mean_length": 12.21875, + "completions/min_length": 2.0, + "epoch": 2.6315789473684212, + "grad_norm": 1.3493712018234787, + "kl": 1.6652233960921876, + "learning_rate": 8.088614830994223e-06, + "loss": 0.015416830778121948, + "memory(GiB)": 79.43, + "reward": 0.1640625, + "reward_std": 0.21537472307682037, + "rewards/Response_no_think_reward_1/mean": 0.1640625, + "rewards/Response_no_think_reward_1/std": 0.7713097929954529, + "step": 237, + "train_speed(iter/s)": 0.007735 + }, + { + "clip_ratio/high_max": 0.0005482456181198359, + "clip_ratio/high_mean": 0.0005482456181198359, + "clip_ratio/low_mean": 0.0018879275157814845, + "clip_ratio/low_min": 0.0018879275157814845, + "clip_ratio/region_mean": 0.0024361731339013204, + "epoch": 2.6426592797783934, + "grad_norm": 1.3201384353784884, + "kl": 1.5714624499087222, + "learning_rate": 7.612046748871327e-06, + "loss": 0.01534443162381649, + "memory(GiB)": 79.43, + "step": 238, + "train_speed(iter/s)": 0.00772 + }, + { + "clip_ratio/high_max": 0.00204707685043104, + "clip_ratio/high_mean": 0.00204707685043104, + "clip_ratio/low_mean": 0.007919068011688069, + "clip_ratio/low_min": 0.007919068011688069, + "clip_ratio/region_mean": 0.009966144862119108, + "completions/clipped_ratio": 0.125, + "completions/max_length": 36.0, + "completions/mean_length": 10.8515625, + "completions/min_length": 2.0, + "epoch": 2.6537396121883656, + "grad_norm": 2.827545573836865, + "kl": 1.8283915198408067, + "learning_rate": 7.149391952678452e-06, + "loss": 0.017831573262810707, + "memory(GiB)": 79.43, + "reward": 0.2890625, + "reward_std": 0.3461739718914032, + "rewards/Response_no_think_reward_1/mean": 0.2890625, + "rewards/Response_no_think_reward_1/std": 0.7852212190628052, + "step": 239, + "train_speed(iter/s)": 0.00762 + }, + { + "clip_ratio/high_max": 0.005028763844165951, + "clip_ratio/high_mean": 0.005028763844165951, + "clip_ratio/low_mean": 0.008577116299420595, + "clip_ratio/low_min": 0.008577116299420595, + "clip_ratio/region_mean": 0.013605880201794207, + "epoch": 2.664819944598338, + "grad_norm": 3.1585454235161925, + "kl": 1.771346734254621, + "learning_rate": 6.700720116526116e-06, + "loss": 0.01860269345343113, + "memory(GiB)": 79.43, + "step": 240, + "train_speed(iter/s)": 0.007606 + }, + { + "clip_ratio/high_max": 0.0017265193164348602, + "clip_ratio/high_mean": 0.0017265193164348602, + "clip_ratio/low_mean": 0.0015245491813402623, + "clip_ratio/low_min": 0.0015245491813402623, + "clip_ratio/region_mean": 0.003251068526878953, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 46.0, + "completions/mean_length": 8.765625, + "completions/min_length": 2.0, + "epoch": 2.67590027700831, + "grad_norm": 2.4712156808930987, + "kl": 3.0323174638469936, + "learning_rate": 6.266098808742516e-06, + "loss": 0.027942461892962456, + "memory(GiB)": 79.43, + "reward": 0.2734375, + "reward_std": 0.09069566428661346, + "rewards/Response_no_think_reward_1/mean": 0.2734375, + "rewards/Response_no_think_reward_1/std": 0.7908416986465454, + "step": 241, + "train_speed(iter/s)": 0.007496 + }, + { + "clip_ratio/high_max": 0.0027624310459941626, + "clip_ratio/high_mean": 0.0027624310459941626, + "clip_ratio/low_mean": 0.006610044481931254, + "clip_ratio/low_min": 0.006610044481931254, + "clip_ratio/region_mean": 0.009372475324198604, + "epoch": 2.6869806094182827, + "grad_norm": 1.4174226924918587, + "kl": 2.842063266711193, + "learning_rate": 5.8455934816979305e-06, + "loss": 0.025462469086050987, + "memory(GiB)": 79.43, + "step": 242, + "train_speed(iter/s)": 0.007483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0027961052255705, + "clip_ratio/low_min": 0.0027961052255705, + "clip_ratio/region_mean": 0.0027961052255705, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 51.0, + "completions/mean_length": 10.0625, + "completions/min_length": 2.0, + "epoch": 2.698060941828255, + "grad_norm": 2.4284263318080272, + "kl": 1.1247594757005572, + "learning_rate": 5.439267461947883e-06, + "loss": 0.013277001678943634, + "memory(GiB)": 79.43, + "reward": 0.4765625, + "reward_std": 0.2019251585006714, + "rewards/Response_no_think_reward_1/mean": 0.4765625, + "rewards/Response_no_think_reward_1/std": 0.8031909465789795, + "step": 243, + "train_speed(iter/s)": 0.007378 + }, + { + "clip_ratio/high_max": 0.00271630787756294, + "clip_ratio/high_mean": 0.00271630787756294, + "clip_ratio/low_mean": 0.003980028559453785, + "clip_ratio/low_min": 0.003980028559453785, + "clip_ratio/region_mean": 0.0066963364370167255, + "epoch": 2.709141274238227, + "grad_norm": 1.999946631431668, + "kl": 1.150102037936449, + "learning_rate": 5.047181940696333e-06, + "loss": 0.011227283626794815, + "memory(GiB)": 79.43, + "step": 244, + "train_speed(iter/s)": 0.007365 + }, + { + "clip_ratio/high_max": 0.003839633078314364, + "clip_ratio/high_mean": 0.003839633078314364, + "clip_ratio/low_mean": 0.002922048792243004, + "clip_ratio/low_min": 0.002922048792243004, + "clip_ratio/region_mean": 0.006761681521311402, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 51.0, + "completions/mean_length": 10.1875, + "completions/min_length": 2.0, + "epoch": 2.7202216066481997, + "grad_norm": 114.02408735149653, + "kl": 4.607287279635784, + "learning_rate": 4.669395964580614e-06, + "loss": 0.09313861280679703, + "memory(GiB)": 79.43, + "reward": 0.265625, + "reward_std": 0.22673699259757996, + "rewards/Response_no_think_reward_1/mean": 0.265625, + "rewards/Response_no_think_reward_1/std": 0.7475352883338928, + "step": 245, + "train_speed(iter/s)": 0.00727 + }, + { + "clip_ratio/high_max": 0.004987254389561713, + "clip_ratio/high_mean": 0.004987254389561713, + "clip_ratio/low_mean": 0.005028058891184628, + "clip_ratio/low_min": 0.005028058891184628, + "clip_ratio/region_mean": 0.01001531328074634, + "completions/clipped_ratio": 0.125, + "completions/max_length": 32.0, + "completions/mean_length": 9.0390625, + "completions/min_length": 2.0, + "epoch": 1.6472060050041701, + "grad_norm": 7.885151261057047, + "kl": 4.317316887667403, + "learning_rate": 0.0001363371892680654, + "loss": 0.06505110114812851, + "memory(GiB)": 77.36, + "reward": 0.390625, + "reward_std": 0.2467075139284134, + "rewards/Response_no_think_reward_1/mean": 0.390625, + "rewards/Response_no_think_reward_1/std": 0.7124801278114319, + "step": 246, + "train_speed(iter/s)": 0.212467 + }, + { + "clip_ratio/high_max": 0.008637543302029371, + "clip_ratio/high_mean": 0.008637543302029371, + "clip_ratio/low_mean": 0.007979447836987674, + "clip_ratio/low_min": 0.007979447836987674, + "clip_ratio/region_mean": 0.016616990906186402, + "epoch": 1.6538782318598833, + "grad_norm": 9.765727829411468, + "kl": 5.302235448267311, + "learning_rate": 0.00013581952149432303, + "loss": 0.07505911588668823, + "memory(GiB)": 77.36, + "step": 247, + "train_speed(iter/s)": 0.182988 + }, + { + "clip_ratio/high_max": 0.00024900399148464203, + "clip_ratio/high_mean": 0.00024900399148464203, + "clip_ratio/low_mean": 0.006602977169677615, + "clip_ratio/low_min": 0.006602977169677615, + "clip_ratio/region_mean": 0.006851981161162257, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 51.0, + "completions/mean_length": 13.0625, + "completions/min_length": 2.0, + "epoch": 1.6605504587155964, + "grad_norm": 5.732775568700692, + "kl": 7.248534864047542, + "learning_rate": 0.0001353007501870137, + "loss": 0.07064881920814514, + "memory(GiB)": 77.47, + "reward": 0.0234375, + "reward_std": 0.21814784407615662, + "rewards/Response_no_think_reward_1/mean": 0.0234375, + "rewards/Response_no_think_reward_1/std": 0.8080777525901794, + "step": 248, + "train_speed(iter/s)": 0.101239 + }, + { + "clip_ratio/high_max": 0.009486032678978518, + "clip_ratio/high_mean": 0.009486032678978518, + "clip_ratio/low_mean": 0.01039744314039126, + "clip_ratio/low_min": 0.01039744314039126, + "clip_ratio/region_mean": 0.019883474858943373, + "epoch": 1.6672226855713093, + "grad_norm": 6.377381125669229, + "kl": 2.9445294842589647, + "learning_rate": 0.00013478089132852716, + "loss": 0.08124249428510666, + "memory(GiB)": 77.47, + "step": 249, + "train_speed(iter/s)": 0.094255 + }, + { + "clip_ratio/high_max": 0.0010964912362396717, + "clip_ratio/high_mean": 0.0010964912362396717, + "clip_ratio/low_mean": 0.003289473708719015, + "clip_ratio/low_min": 0.003289473708719015, + "clip_ratio/region_mean": 0.004385964944958687, + "completions/clipped_ratio": 0.125, + "completions/max_length": 51.0, + "completions/mean_length": 7.9140625, + "completions/min_length": 2.0, + "epoch": 1.6738949124270226, + "grad_norm": 1.96116793666507, + "kl": 2.669485174041256, + "learning_rate": 0.0001342599609347585, + "loss": 0.02923566661775112, + "memory(GiB)": 77.47, + "reward": -0.0625, + "reward_std": 0.09127141535282135, + "rewards/Response_no_think_reward_1/mean": -0.0625, + "rewards/Response_no_think_reward_1/std": 0.9200308322906494, + "step": 250, + "train_speed(iter/s)": 0.065352 + }, + { + "clip_ratio/high_max": 0.005629746010527015, + "clip_ratio/high_mean": 0.005629746010527015, + "clip_ratio/low_mean": 0.0412575276568532, + "clip_ratio/low_min": 0.0412575276568532, + "clip_ratio/region_mean": 0.04688727576285601, + "epoch": 1.6805671392827355, + "grad_norm": 1.933812668241347, + "kl": 3.5041955505930673, + "learning_rate": 0.000133737975054615, + "loss": 0.024076396599411964, + "memory(GiB)": 79.43, + "step": 251, + "train_speed(iter/s)": 0.062446 + }, + { + "clip_ratio/high_max": 0.00119566879584454, + "clip_ratio/high_mean": 0.00119566879584454, + "clip_ratio/low_mean": 0.0004687021573772654, + "clip_ratio/low_min": 0.0004687021573772654, + "clip_ratio/region_mean": 0.0016643709386698902, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 51.0, + "completions/mean_length": 12.6953125, + "completions/min_length": 2.0, + "epoch": 1.6872393661384488, + "grad_norm": 2.4412702069582983, + "kl": 3.9371446361765265, + "learning_rate": 0.00013321494976952132, + "loss": 0.03714549541473389, + "memory(GiB)": 79.43, + "reward": 0.1796875, + "reward_std": 0.13782459497451782, + "rewards/Response_no_think_reward_1/mean": 0.1796875, + "rewards/Response_no_think_reward_1/std": 0.9258628487586975, + "step": 252, + "train_speed(iter/s)": 0.047385 + }, + { + "clip_ratio/high_max": 0.007317493204027414, + "clip_ratio/high_mean": 0.007317493204027414, + "clip_ratio/low_mean": 0.005186333088204265, + "clip_ratio/low_min": 0.005186333088204265, + "clip_ratio/region_mean": 0.012503825942985713, + "epoch": 1.6939115929941617, + "grad_norm": 4.580377931462122, + "kl": 1.7039022920653224, + "learning_rate": 0.00013269090119292444, + "loss": 0.021267667412757874, + "memory(GiB)": 79.43, + "step": 253, + "train_speed(iter/s)": 0.045915 + }, + { + "clip_ratio/high_max": 0.0008462663099635392, + "clip_ratio/high_mean": 0.0008462663099635392, + "clip_ratio/low_mean": 0.005740528285969049, + "clip_ratio/low_min": 0.005740528285969049, + "clip_ratio/region_mean": 0.006586794595932588, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 51.0, + "completions/mean_length": 13.03125, + "completions/min_length": 2.0, + "epoch": 1.700583819849875, + "grad_norm": 2.9913603629745245, + "kl": 1.2227803394198418, + "learning_rate": 0.00013216584546979702, + "loss": 0.01559203490614891, + "memory(GiB)": 79.43, + "reward": 0.015625, + "reward_std": 0.27663108706474304, + "rewards/Response_no_think_reward_1/mean": 0.015625, + "rewards/Response_no_think_reward_1/std": 0.9555834531784058, + "step": 254, + "train_speed(iter/s)": 0.037104 + }, + { + "clip_ratio/high_max": 0.007223855180200189, + "clip_ratio/high_mean": 0.007223855180200189, + "clip_ratio/low_mean": 0.10893735339050181, + "clip_ratio/low_min": 0.10893735339050181, + "clip_ratio/region_mean": 0.11616120918188244, + "epoch": 1.707256046705588, + "grad_norm": 1.9655670074579659, + "kl": 2.0845303861424327, + "learning_rate": 0.00013163979877614, + "loss": 0.011529060080647469, + "memory(GiB)": 79.43, + "step": 255, + "train_speed(iter/s)": 0.036232 + }, + { + "clip_ratio/high_max": 0.006948165129870176, + "clip_ratio/high_mean": 0.006948165129870176, + "clip_ratio/low_mean": 0.00925130266114138, + "clip_ratio/low_min": 0.00925130266114138, + "clip_ratio/region_mean": 0.01619946787832305, + "completions/clipped_ratio": 0.359375, + "completions/max_length": 51.0, + "completions/mean_length": 14.140625, + "completions/min_length": 2.0, + "epoch": 1.713928273561301, + "grad_norm": 4.730855130110655, + "kl": 5.062177191488445, + "learning_rate": 0.00013111277731848442, + "loss": 0.05786250904202461, + "memory(GiB)": 79.43, + "reward": -0.0859375, + "reward_std": 0.410529226064682, + "rewards/Response_no_think_reward_1/mean": -0.0859375, + "rewards/Response_no_think_reward_1/std": 0.869675874710083, + "step": 256, + "train_speed(iter/s)": 0.033266 + }, + { + "clip_ratio/high_max": 0.014393029327038676, + "clip_ratio/high_mean": 0.014393029327038676, + "clip_ratio/low_mean": 0.1303235706873238, + "clip_ratio/low_min": 0.1303235706873238, + "clip_ratio/region_mean": 0.14471660286653787, + "epoch": 1.7206005004170142, + "grad_norm": 6.110613692874304, + "kl": 6.920241659507155, + "learning_rate": 0.00013058479733339185, + "loss": 0.0579487606883049, + "memory(GiB)": 79.43, + "step": 257, + "train_speed(iter/s)": 0.032582 + }, + { + "clip_ratio/high_max": 0.0021560846944339573, + "clip_ratio/high_mean": 0.0021560846944339573, + "clip_ratio/low_mean": 0.027111168601550162, + "clip_ratio/low_min": 0.027111168601550162, + "clip_ratio/region_mean": 0.029267253237776458, + "completions/clipped_ratio": 0.296875, + "completions/max_length": 51.0, + "completions/mean_length": 9.203125, + "completions/min_length": 2.0, + "epoch": 1.7272727272727273, + "grad_norm": 13.319715864612775, + "kl": 13.24667002509068, + "learning_rate": 0.00013005587508695444, + "loss": 0.12780021131038666, + "memory(GiB)": 79.43, + "reward": 0.109375, + "reward_std": 0.3405221104621887, + "rewards/Response_no_think_reward_1/mean": 0.109375, + "rewards/Response_no_think_reward_1/std": 0.8982430100440979, + "step": 258, + "train_speed(iter/s)": 0.030335 + }, + { + "clip_ratio/high_max": 0.025854239764157683, + "clip_ratio/high_mean": 0.025854239764157683, + "clip_ratio/low_mean": 0.02940721553750336, + "clip_ratio/low_min": 0.02940721553750336, + "clip_ratio/region_mean": 0.055261454777792096, + "epoch": 1.7339449541284404, + "grad_norm": 7.873376409391432, + "kl": 3.398330974082228, + "learning_rate": 0.00012952602687429362, + "loss": 0.08237586915493011, + "memory(GiB)": 79.43, + "step": 259, + "train_speed(iter/s)": 0.029779 + }, + { + "clip_ratio/high_max": 0.0010593220358714461, + "clip_ratio/high_mean": 0.0010593220358714461, + "clip_ratio/low_mean": 0.025134949712082744, + "clip_ratio/low_min": 0.025134949712082744, + "clip_ratio/region_mean": 0.02619427174795419, + "completions/clipped_ratio": 0.3828125, + "completions/max_length": 51.0, + "completions/mean_length": 11.0390625, + "completions/min_length": 2.0, + "epoch": 1.7406171809841533, + "grad_norm": 5.622914024348242, + "kl": 8.146455611102283, + "learning_rate": 0.00012899526901905822, + "loss": 0.09567078202962875, + "memory(GiB)": 79.43, + "reward": 0.046875, + "reward_std": 0.30238547921180725, + "rewards/Response_no_think_reward_1/mean": 0.046875, + "rewards/Response_no_think_reward_1/std": 0.8681537508964539, + "step": 260, + "train_speed(iter/s)": 0.028068 + }, + { + "clip_ratio/high_max": 0.01715171878458932, + "clip_ratio/high_mean": 0.01715171878458932, + "clip_ratio/low_mean": 0.0735002284636721, + "clip_ratio/low_min": 0.0735002284636721, + "clip_ratio/region_mean": 0.0906519484706223, + "epoch": 1.7472894078398666, + "grad_norm": 8.808673951478834, + "kl": 5.425452136900276, + "learning_rate": 0.00012846361787292136, + "loss": 0.08871767669916153, + "memory(GiB)": 79.43, + "step": 261, + "train_speed(iter/s)": 0.027598 + }, + { + "clip_ratio/high_max": 0.000811688310932368, + "clip_ratio/high_mean": 0.000811688310932368, + "clip_ratio/low_mean": 0.009334841219242662, + "clip_ratio/low_min": 0.009334841219242662, + "clip_ratio/region_mean": 0.01014652947196737, + "completions/clipped_ratio": 0.3984375, + "completions/max_length": 51.0, + "completions/mean_length": 10.890625, + "completions/min_length": 2.0, + "epoch": 1.7539616346955795, + "grad_norm": 9.615584503884596, + "kl": 8.083851436153054, + "learning_rate": 0.00012793108981507694, + "loss": 0.11578933149576187, + "memory(GiB)": 79.43, + "reward": -0.0859375, + "reward_std": 0.41268494725227356, + "rewards/Response_no_think_reward_1/mean": -0.0859375, + "rewards/Response_no_think_reward_1/std": 0.8605742454528809, + "step": 262, + "train_speed(iter/s)": 0.02625 + }, + { + "clip_ratio/high_max": 0.020523510698694736, + "clip_ratio/high_mean": 0.020523510698694736, + "clip_ratio/low_mean": 0.04563906986732036, + "clip_ratio/low_min": 0.04563906986732036, + "clip_ratio/region_mean": 0.0661625819047913, + "epoch": 1.7606338615512929, + "grad_norm": 30.924872115013905, + "kl": 3.7772719897329807, + "learning_rate": 0.0001273977012517348, + "loss": 0.21410413086414337, + "memory(GiB)": 79.43, + "step": 263, + "train_speed(iter/s)": 0.025852 + }, + { + "clip_ratio/high_max": 0.0010416667209938169, + "clip_ratio/high_mean": 0.0010416667209938169, + "clip_ratio/low_mean": 0.013979690615087748, + "clip_ratio/low_min": 0.013979690615087748, + "clip_ratio/region_mean": 0.015021357336081564, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 41.0, + "completions/mean_length": 7.9609375, + "completions/min_length": 2.0, + "epoch": 1.7673060884070058, + "grad_norm": 2.801515232228881, + "kl": 1.5485301897861063, + "learning_rate": 0.00012686346861561537, + "loss": 0.014268179424107075, + "memory(GiB)": 79.43, + "reward": 0.015625, + "reward_std": 0.2223242074251175, + "rewards/Response_no_think_reward_1/mean": 0.015625, + "rewards/Response_no_think_reward_1/std": 0.8227510452270508, + "step": 264, + "train_speed(iter/s)": 0.024697 + }, + { + "clip_ratio/high_max": 0.0018305982230231166, + "clip_ratio/high_mean": 0.0018305982230231166, + "clip_ratio/low_mean": 0.0754491506377235, + "clip_ratio/low_min": 0.0754491506377235, + "clip_ratio/region_mean": 0.07727974897716194, + "epoch": 1.773978315262719, + "grad_norm": 5.020289308786792, + "kl": 4.038693455513567, + "learning_rate": 0.0001263284083654435, + "loss": 0.035946328192949295, + "memory(GiB)": 79.43, + "step": 265, + "train_speed(iter/s)": 0.024353 + }, + { + "clip_ratio/high_max": 0.0015625000232830644, + "clip_ratio/high_mean": 0.0015625000232830644, + "clip_ratio/low_mean": 0.011535812984220684, + "clip_ratio/low_min": 0.011535812984220684, + "clip_ratio/region_mean": 0.013098313007503748, + "completions/clipped_ratio": 0.2421875, + "completions/max_length": 51.0, + "completions/mean_length": 7.0546875, + "completions/min_length": 2.0, + "epoch": 1.780650542118432, + "grad_norm": 4.823867965705569, + "kl": 5.8212087393421825, + "learning_rate": 0.00012579253698544125, + "loss": 0.07973380386829376, + "memory(GiB)": 79.43, + "reward": 0.0078125, + "reward_std": 0.30061954259872437, + "rewards/Response_no_think_reward_1/mean": 0.0078125, + "rewards/Response_no_think_reward_1/std": 0.6459577083587646, + "step": 266, + "train_speed(iter/s)": 0.023359 + }, + { + "clip_ratio/high_max": 0.0031250000465661287, + "clip_ratio/high_mean": 0.0031250000465661287, + "clip_ratio/low_mean": 0.11317632021382451, + "clip_ratio/low_min": 0.11317632021382451, + "clip_ratio/region_mean": 0.11630132002756, + "epoch": 1.787322768974145, + "grad_norm": 11.546843380663908, + "kl": 10.476074979640543, + "learning_rate": 0.00012525587098482005, + "loss": 0.09422159940004349, + "memory(GiB)": 79.43, + "step": 267, + "train_speed(iter/s)": 0.023058 + }, + { + "clip_ratio/high_max": 0.000811688310932368, + "clip_ratio/high_mean": 0.000811688310932368, + "clip_ratio/low_mean": 0.006867786491056904, + "clip_ratio/low_min": 0.006867786491056904, + "clip_ratio/region_mean": 0.007679474772885442, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 51.0, + "completions/mean_length": 9.8515625, + "completions/min_length": 3.0, + "epoch": 1.7939949958298582, + "grad_norm": 48.055640901825676, + "kl": 22.36742792606401, + "learning_rate": 0.0001247184268972722, + "loss": 0.4101108908653259, + "memory(GiB)": 79.43, + "reward": 0.125, + "reward_std": 0.2585597634315491, + "rewards/Response_no_think_reward_1/mean": 0.125, + "rewards/Response_no_think_reward_1/std": 0.6274557709693909, + "step": 268, + "train_speed(iter/s)": 0.022062 + }, + { + "clip_ratio/high_max": 0.00529984722379595, + "clip_ratio/high_mean": 0.00529984722379595, + "clip_ratio/low_mean": 0.01420470466837287, + "clip_ratio/low_min": 0.01420470466837287, + "clip_ratio/region_mean": 0.01950455189216882, + "epoch": 1.8006672226855713, + "grad_norm": 10.145223773494124, + "kl": 7.905034697459087, + "learning_rate": 0.00012418022128046142, + "loss": 0.22370362281799316, + "memory(GiB)": 79.43, + "step": 269, + "train_speed(iter/s)": 0.021799 + }, + { + "clip_ratio/high_max": 0.0020380434580147266, + "clip_ratio/high_mean": 0.0020380434580147266, + "clip_ratio/low_mean": 0.004399414756335318, + "clip_ratio/low_min": 0.004399414756335318, + "clip_ratio/region_mean": 0.006437458097934723, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 51.0, + "completions/mean_length": 9.84375, + "completions/min_length": 3.0, + "epoch": 1.8073394495412844, + "grad_norm": 2.925526379332068, + "kl": 1.9621957493945956, + "learning_rate": 0.0001236412707155127, + "loss": 0.02435774728655815, + "memory(GiB)": 79.43, + "reward": 0.34375, + "reward_std": 0.22380223870277405, + "rewards/Response_no_think_reward_1/mean": 0.34375, + "rewards/Response_no_think_reward_1/std": 0.6329222321510315, + "step": 270, + "train_speed(iter/s)": 0.021038 + }, + { + "clip_ratio/high_max": 0.027234246022999287, + "clip_ratio/high_mean": 0.027234246022999287, + "clip_ratio/low_mean": 0.014790827757678926, + "clip_ratio/low_min": 0.014790827757678926, + "clip_ratio/region_mean": 0.04202507343143225, + "epoch": 1.8140116763969973, + "grad_norm": 7.85150802599005, + "kl": 1.2954727746546268, + "learning_rate": 0.0001231015918065016, + "loss": 0.046614713966846466, + "memory(GiB)": 79.43, + "step": 271, + "train_speed(iter/s)": 0.020801 + }, + { + "clip_ratio/high_max": 0.0007812500116415322, + "clip_ratio/high_mean": 0.0007812500116415322, + "clip_ratio/low_mean": 0.008673313190229237, + "clip_ratio/low_min": 0.008673313190229237, + "clip_ratio/region_mean": 0.00945456320187077, + "completions/clipped_ratio": 0.125, + "completions/max_length": 23.0, + "completions/mean_length": 8.578125, + "completions/min_length": 2.0, + "epoch": 1.8206839032527107, + "grad_norm": 5.663376365839488, + "kl": 3.0526226649526507, + "learning_rate": 0.00012256120117994245, + "loss": 0.029910365119576454, + "memory(GiB)": 79.43, + "reward": 0.2578125, + "reward_std": 0.2626354694366455, + "rewards/Response_no_think_reward_1/mean": 0.2578125, + "rewards/Response_no_think_reward_1/std": 0.70147305727005, + "step": 272, + "train_speed(iter/s)": 0.020032 + }, + { + "clip_ratio/high_max": 0.003040286072064191, + "clip_ratio/high_mean": 0.003040286072064191, + "clip_ratio/low_mean": 0.06708688009530306, + "clip_ratio/low_min": 0.06708688009530306, + "clip_ratio/region_mean": 0.07012716663302854, + "epoch": 1.8273561301084236, + "grad_norm": 4.421434755408977, + "kl": 5.31667997315526, + "learning_rate": 0.0001220201154842765, + "loss": 0.03301015496253967, + "memory(GiB)": 79.43, + "step": 273, + "train_speed(iter/s)": 0.019826 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.007332374923862517, + "clip_ratio/low_min": 0.007332374923862517, + "clip_ratio/region_mean": 0.007332374923862517, + "completions/clipped_ratio": 0.1796875, + "completions/max_length": 51.0, + "completions/mean_length": 11.125, + "completions/min_length": 2.0, + "epoch": 1.834028356964137, + "grad_norm": 1.9255495711888, + "kl": 2.3658070964738727, + "learning_rate": 0.00012147835138935868, + "loss": 0.040864668786525726, + "memory(GiB)": 79.43, + "reward": -0.3203125, + "reward_std": 0.22869102656841278, + "rewards/Response_no_think_reward_1/mean": -0.3203125, + "rewards/Response_no_think_reward_1/std": 0.7091482281684875, + "step": 274, + "train_speed(iter/s)": 0.019168 + }, + { + "clip_ratio/high_max": 0.014975278521887958, + "clip_ratio/high_mean": 0.014975278521887958, + "clip_ratio/low_mean": 0.056431527715176344, + "clip_ratio/low_min": 0.056431527715176344, + "clip_ratio/region_mean": 0.07140680588781834, + "epoch": 1.8407005838198498, + "grad_norm": 1.7607705528260904, + "kl": 2.3127799107460305, + "learning_rate": 0.00012093592558594416, + "loss": 0.021540286019444466, + "memory(GiB)": 79.43, + "step": 275, + "train_speed(iter/s)": 0.018983 + }, + { + "clip_ratio/high_max": 0.00027173911803402007, + "clip_ratio/high_mean": 0.00027173911803402007, + "clip_ratio/low_mean": 0.0030319468351081014, + "clip_ratio/low_min": 0.0030319468351081014, + "clip_ratio/region_mean": 0.003303685924038291, + "completions/clipped_ratio": 0.1171875, + "completions/max_length": 51.0, + "completions/mean_length": 10.21875, + "completions/min_length": 5.0, + "epoch": 1.847372810675563, + "grad_norm": 1.3845096214977168, + "kl": 0.5768661912588868, + "learning_rate": 0.00012039285478517417, + "loss": 0.005703174974769354, + "memory(GiB)": 79.43, + "reward": 0.296875, + "reward_std": 0.31093141436576843, + "rewards/Response_no_think_reward_1/mean": 0.296875, + "rewards/Response_no_think_reward_1/std": 0.6563964486122131, + "step": 276, + "train_speed(iter/s)": 0.018355 + }, + { + "clip_ratio/high_max": 0.02822316304082051, + "clip_ratio/high_mean": 0.02822316304082051, + "clip_ratio/low_mean": 0.023227162193506956, + "clip_ratio/low_min": 0.023227162193506956, + "clip_ratio/region_mean": 0.05145032424479723, + "epoch": 1.854045037531276, + "grad_norm": 2.277051850874954, + "kl": 1.444888403537334, + "learning_rate": 0.00011984915571806108, + "loss": 0.004672436509281397, + "memory(GiB)": 79.43, + "step": 277, + "train_speed(iter/s)": 0.018189 + }, + { + "clip_ratio/high_max": 0.006991013826336712, + "clip_ratio/high_mean": 0.006991013826336712, + "clip_ratio/low_mean": 0.009722222341224551, + "clip_ratio/low_min": 0.009722222341224551, + "clip_ratio/region_mean": 0.016713236400391906, + "completions/clipped_ratio": 0.0703125, + "completions/max_length": 25.0, + "completions/mean_length": 6.9765625, + "completions/min_length": 2.0, + "epoch": 1.8607172643869891, + "grad_norm": 35.77457650983362, + "kl": 31.698298294795677, + "learning_rate": 0.000119304845134973, + "loss": 0.27273401618003845, + "memory(GiB)": 79.43, + "reward": 0.1015625, + "reward_std": 0.17464229464530945, + "rewards/Response_no_think_reward_1/mean": 0.1015625, + "rewards/Response_no_think_reward_1/std": 0.6621350049972534, + "step": 278, + "train_speed(iter/s)": 0.017652 + }, + { + "clip_ratio/high_max": 0.009891506866551936, + "clip_ratio/high_mean": 0.009891506866551936, + "clip_ratio/low_mean": 0.0069444444961845875, + "clip_ratio/low_min": 0.0069444444961845875, + "clip_ratio/region_mean": 0.016835951362736523, + "epoch": 1.8673894912427023, + "grad_norm": 4.542577039630815, + "kl": 3.7718499208567664, + "learning_rate": 0.00011875993980511773, + "loss": 0.07798631489276886, + "memory(GiB)": 79.43, + "step": 279, + "train_speed(iter/s)": 0.017502 + }, + { + "clip_ratio/high_max": 0.002536526066251099, + "clip_ratio/high_mean": 0.002536526066251099, + "clip_ratio/low_mean": 0.0024038462433964014, + "clip_ratio/low_min": 0.0024038462433964014, + "clip_ratio/region_mean": 0.0049403723096475005, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 27.0, + "completions/mean_length": 8.1484375, + "completions/min_length": 2.0, + "epoch": 1.8740617180984154, + "grad_norm": 2.8417572027389006, + "kl": 3.5999494855445846, + "learning_rate": 0.00011821445651602616, + "loss": 0.038278549909591675, + "memory(GiB)": 79.43, + "reward": 0.078125, + "reward_std": 0.18506525456905365, + "rewards/Response_no_think_reward_1/mean": 0.078125, + "rewards/Response_no_think_reward_1/std": 0.7797574400901794, + "step": 280, + "train_speed(iter/s)": 0.016994 + } + ], + "logging_steps": 1, + "max_steps": 596, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 5, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}