{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.08571918395336876, "eval_steps": 500, "global_step": 500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.009895833674818277, "completions/max_length": 1024.0, "completions/max_terminated_length": 921.2, "completions/mean_length": 747.1151245117187, "completions/mean_terminated_length": 744.3689392089843, "completions/min_length": 457.8, "completions/min_terminated_length": 457.8, "entropy": 0.7938701828320821, "epoch": 0.0017143836790673753, "frac_reward_zero_std": 0.0, "grad_norm": 0.294921875, "learning_rate": 1e-06, "loss": 0.0024, "num_tokens": 4482493.0, "reward": -11.88613681793213, "reward_std": 9.975114345550537, "rewards/ADERawReward/mean": -11.914521789550781, "rewards/ADERawReward/std": 9.975466585159301, "rewards/StrictFormatReward/mean": 0.28385417386889455, "rewards/StrictFormatReward/std": 2.689612889289856, "sampling/importance_sampling_ratio/max": 2.7710394859313965, "sampling/importance_sampling_ratio/mean": 0.38329996466636657, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.348051142692566, "sampling/sampling_logp_difference/mean": 0.02634742669761181, "step": 10, "step_time": 31.851699317700696 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.014062500465661288, "completions/max_length": 1024.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 746.5093994140625, "completions/mean_terminated_length": 742.5637084960938, "completions/min_length": 319.8, "completions/min_terminated_length": 319.8, "entropy": 0.783385141690572, "epoch": 0.0034287673581347507, "frac_reward_zero_std": 0.0, "grad_norm": 0.341796875, "learning_rate": 1e-06, "loss": -0.0167, "num_tokens": 8963695.0, "reward": -10.739660549163819, "reward_std": 7.596912527084351, "rewards/ADERawReward/mean": -10.76804552078247, "rewards/ADERawReward/std": 7.586498594284057, "rewards/StrictFormatReward/mean": 0.2838541708886623, "rewards/StrictFormatReward/std": 2.69962215423584, "sampling/importance_sampling_ratio/max": 2.778630328178406, "sampling/importance_sampling_ratio/mean": 0.38422776758670807, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.3024064064025875, "sampling/sampling_logp_difference/mean": 0.026079285889863968, "step": 20, "step_time": 31.08811371029442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01302083358168602, "completions/max_length": 1024.0, "completions/max_terminated_length": 916.5, "completions/mean_length": 745.9713745117188, "completions/mean_terminated_length": 742.3107482910157, "completions/min_length": 344.2, "completions/min_terminated_length": 344.2, "entropy": 0.777154250939687, "epoch": 0.005143151037202126, "frac_reward_zero_std": 0.0, "grad_norm": 0.240234375, "learning_rate": 1e-06, "loss": -0.0033, "num_tokens": 13443080.0, "reward": -10.726643276214599, "reward_std": 6.941620349884033, "rewards/ADERawReward/mean": -10.759612274169921, "rewards/ADERawReward/std": 6.939747667312622, "rewards/StrictFormatReward/mean": 0.32968750670552255, "rewards/StrictFormatReward/std": 2.6174988746643066, "sampling/importance_sampling_ratio/max": 2.820967411994934, "sampling/importance_sampling_ratio/mean": 0.3970964789390564, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.230282878875732, "sampling/sampling_logp_difference/mean": 0.025640238262712955, "step": 30, "step_time": 31.281337887697738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.006250000186264515, "completions/max_length": 1006.1, "completions/max_terminated_length": 952.6, "completions/mean_length": 746.8578247070312, "completions/mean_terminated_length": 745.1074523925781, "completions/min_length": 574.6, "completions/min_terminated_length": 574.6, "entropy": 0.7786477545897166, "epoch": 0.006857534716269501, "frac_reward_zero_std": 0.0, "grad_norm": 0.353515625, "learning_rate": 1e-06, "loss": 0.0052, "num_tokens": 17925079.0, "reward": -10.400494289398193, "reward_std": 8.363166570663452, "rewards/ADERawReward/mean": -10.453514766693115, "rewards/ADERawReward/std": 8.35603289604187, "rewards/StrictFormatReward/mean": 0.5302083313465118, "rewards/StrictFormatReward/std": 2.1997122406959533, "sampling/importance_sampling_ratio/max": 2.7520124673843385, "sampling/importance_sampling_ratio/mean": 0.4182109236717224, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.882514882087707, "sampling/sampling_logp_difference/mean": 0.025688859820365905, "step": 40, "step_time": 30.94830545460136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.010416666883975268, "completions/max_length": 1008.7, "completions/max_terminated_length": 908.0, "completions/mean_length": 744.6708557128907, "completions/mean_terminated_length": 741.7320251464844, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.7797636012236278, "epoch": 0.008571918395336876, "frac_reward_zero_std": 0.0, "grad_norm": 0.296875, "learning_rate": 1e-06, "loss": 0.0059, "num_tokens": 22402927.0, "reward": -11.912008953094482, "reward_std": 25.465690279006957, "rewards/ADERawReward/mean": -11.948414325714111, "rewards/ADERawReward/std": 25.452851724624633, "rewards/StrictFormatReward/mean": 0.3640625074505806, "rewards/StrictFormatReward/std": 2.5515830755233764, "sampling/importance_sampling_ratio/max": 2.7972999811172485, "sampling/importance_sampling_ratio/mean": 0.41742126941680907, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.1138537406921385, "sampling/sampling_logp_difference/mean": 0.025952593609690666, "step": 50, "step_time": 30.99103705089656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00572916679084301, "completions/max_length": 982.4, "completions/max_terminated_length": 894.4, "completions/mean_length": 743.1708618164063, "completions/mean_terminated_length": 741.5553588867188, "completions/min_length": 389.5, "completions/min_terminated_length": 389.5, "entropy": 0.7705018659432729, "epoch": 0.010286302074404252, "frac_reward_zero_std": 0.0, "grad_norm": 0.294921875, "learning_rate": 1e-06, "loss": -0.001, "num_tokens": 26877415.0, "reward": -10.354321956634521, "reward_std": 7.085966682434082, "rewards/ADERawReward/mean": -10.3998948097229, "rewards/ADERawReward/std": 7.078661155700684, "rewards/StrictFormatReward/mean": 0.45572916492819787, "rewards/StrictFormatReward/std": 2.357165718078613, "sampling/importance_sampling_ratio/max": 2.8156795501708984, "sampling/importance_sampling_ratio/mean": 0.4263548344373703, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.213912606239319, "sampling/sampling_logp_difference/mean": 0.025748718157410623, "step": 60, "step_time": 30.629806772600567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.007812500139698387, "completions/max_length": 997.5, "completions/max_terminated_length": 901.5, "completions/mean_length": 745.1635559082031, "completions/mean_terminated_length": 742.9772766113281, "completions/min_length": 448.0, "completions/min_terminated_length": 448.0, "entropy": 0.7808759689331055, "epoch": 0.012000685753471628, "frac_reward_zero_std": 0.0, "grad_norm": 0.416015625, "learning_rate": 1e-06, "loss": -0.0022, "num_tokens": 31355793.0, "reward": -10.355865859985352, "reward_std": 7.629749822616577, "rewards/ADERawReward/mean": -10.399147129058838, "rewards/ADERawReward/std": 7.623885774612427, "rewards/StrictFormatReward/mean": 0.43281250447034836, "rewards/StrictFormatReward/std": 2.4088792324066164, "sampling/importance_sampling_ratio/max": 2.7858126163482666, "sampling/importance_sampling_ratio/mean": 0.4104765444993973, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.654694700241089, "sampling/sampling_logp_difference/mean": 0.02596179526299238, "step": 70, "step_time": 30.90157012869895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.006250000139698386, "completions/max_length": 997.0, "completions/max_terminated_length": 911.7, "completions/mean_length": 743.9010620117188, "completions/mean_terminated_length": 742.1458801269531, "completions/min_length": 506.5, "completions/min_terminated_length": 506.5, "entropy": 0.7690683722496032, "epoch": 0.013715069432539003, "frac_reward_zero_std": 0.0, "grad_norm": 0.259765625, "learning_rate": 1e-06, "loss": -0.0147, "num_tokens": 35831699.0, "reward": -9.603075504302979, "reward_std": 7.397923803329467, "rewards/ADERawReward/mean": -9.64406452178955, "rewards/ADERawReward/std": 7.394052314758301, "rewards/StrictFormatReward/mean": 0.40989583656191825, "rewards/StrictFormatReward/std": 2.446663808822632, "sampling/importance_sampling_ratio/max": 2.8509432315826415, "sampling/importance_sampling_ratio/mean": 0.42068196535110475, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.871529459953308, "sampling/sampling_logp_difference/mean": 0.025504560954868794, "step": 80, "step_time": 30.775000178898335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.006770833488553762, "completions/max_length": 1018.6, "completions/max_terminated_length": 921.4, "completions/mean_length": 743.0687744140625, "completions/mean_terminated_length": 741.1454040527344, "completions/min_length": 413.2, "completions/min_terminated_length": 413.2, "entropy": 0.7730992416540782, "epoch": 0.015429453111606378, "frac_reward_zero_std": 0.0, "grad_norm": 0.330078125, "learning_rate": 1e-06, "loss": -0.0209, "num_tokens": 40306439.0, "reward": -9.999512577056885, "reward_std": 6.795107650756836, "rewards/ADERawReward/mean": -10.04222068786621, "rewards/ADERawReward/std": 6.785987043380738, "rewards/StrictFormatReward/mean": 0.4270833358168602, "rewards/StrictFormatReward/std": 2.427087187767029, "sampling/importance_sampling_ratio/max": 2.806507611274719, "sampling/importance_sampling_ratio/mean": 0.41283826231956483, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.893343424797058, "sampling/sampling_logp_difference/mean": 0.02597447969019413, "step": 90, "step_time": 31.03075063330034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.006250000139698386, "completions/max_length": 989.1, "completions/max_terminated_length": 911.9, "completions/mean_length": 744.7958557128907, "completions/mean_terminated_length": 743.0538269042969, "completions/min_length": 479.0, "completions/min_terminated_length": 479.0, "entropy": 0.7617896695931753, "epoch": 0.01714383679067375, "frac_reward_zero_std": 0.0, "grad_norm": 0.26953125, "learning_rate": 1e-06, "loss": 0.0056, "num_tokens": 44785151.0, "reward": -10.150122165679932, "reward_std": 6.947455310821534, "rewards/ADERawReward/mean": -10.1985595703125, "rewards/ADERawReward/std": 6.9321370124816895, "rewards/StrictFormatReward/mean": 0.48437499925494193, "rewards/StrictFormatReward/std": 2.2780819296836854, "sampling/importance_sampling_ratio/max": 2.8471882343292236, "sampling/importance_sampling_ratio/mean": 0.4186240643262863, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.192489242553711, "sampling/sampling_logp_difference/mean": 0.025915385968983174, "step": 100, "step_time": 30.777787425796852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.008854166837409139, "completions/max_length": 1016.1, "completions/max_terminated_length": 927.6, "completions/mean_length": 747.430224609375, "completions/mean_terminated_length": 744.9568664550782, "completions/min_length": 542.5, "completions/min_terminated_length": 542.5, "entropy": 0.7839122792085012, "epoch": 0.018858220469741126, "frac_reward_zero_std": 0.0, "grad_norm": 0.369140625, "learning_rate": 1e-06, "loss": 0.0095, "num_tokens": 49267497.0, "reward": -10.34236192703247, "reward_std": 6.522932863235473, "rewards/ADERawReward/mean": -10.377049255371094, "rewards/ADERawReward/std": 6.512392950057984, "rewards/StrictFormatReward/mean": 0.3468750029802322, "rewards/StrictFormatReward/std": 2.5984415769577027, "sampling/importance_sampling_ratio/max": 2.919663596153259, "sampling/importance_sampling_ratio/mean": 0.42760642170906066, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.582067346572876, "sampling/sampling_logp_difference/mean": 0.025711694732308388, "step": 110, "step_time": 31.13533423260669 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.006250000139698386, "completions/max_length": 981.8, "completions/max_terminated_length": 905.6, "completions/mean_length": 741.3906433105469, "completions/mean_terminated_length": 739.6170837402344, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.7655617932478587, "epoch": 0.020572604148808505, "frac_reward_zero_std": 0.0, "grad_norm": 0.2255859375, "learning_rate": 1e-06, "loss": -0.0003, "num_tokens": 53739095.0, "reward": -9.396285820007325, "reward_std": 6.019436550140381, "rewards/ADERawReward/mean": -9.43211908340454, "rewards/ADERawReward/std": 5.99921350479126, "rewards/StrictFormatReward/mean": 0.35833333134651185, "rewards/StrictFormatReward/std": 2.565337634086609, "sampling/importance_sampling_ratio/max": 2.7736236810684205, "sampling/importance_sampling_ratio/mean": 0.4329004347324371, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.103514838218689, "sampling/sampling_logp_difference/mean": 0.02581946086138487, "step": 120, "step_time": 30.675611695800036 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.006250000139698386, "completions/max_length": 993.0, "completions/max_terminated_length": 917.3, "completions/mean_length": 743.2172119140625, "completions/mean_terminated_length": 741.4531127929688, "completions/min_length": 448.6, "completions/min_terminated_length": 448.6, "entropy": 0.7609198371569316, "epoch": 0.02228698782787588, "frac_reward_zero_std": 0.0, "grad_norm": 0.306640625, "learning_rate": 1e-06, "loss": -0.0186, "num_tokens": 58213992.0, "reward": -10.035307788848877, "reward_std": 7.6791582107543945, "rewards/ADERawReward/mean": -10.080307579040527, "rewards/ADERawReward/std": 7.661759281158448, "rewards/StrictFormatReward/mean": 0.45, "rewards/StrictFormatReward/std": 2.383046817779541, "sampling/importance_sampling_ratio/max": 2.7497065782547, "sampling/importance_sampling_ratio/mean": 0.38673609495162964, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.264748442173004, "sampling/sampling_logp_difference/mean": 0.02555535715073347, "step": 130, "step_time": 30.85125565490016 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.008333333535119891, "completions/max_length": 995.9, "completions/max_terminated_length": 936.1, "completions/mean_length": 747.1015808105469, "completions/mean_terminated_length": 744.7781860351563, "completions/min_length": 504.1, "completions/min_terminated_length": 504.1, "entropy": 0.7680625875790914, "epoch": 0.024001371506943255, "frac_reward_zero_std": 0.0, "grad_norm": 0.283203125, "learning_rate": 1e-06, "loss": -0.014, "num_tokens": 62696459.0, "reward": -9.6462965965271, "reward_std": 6.933882856369019, "rewards/ADERawReward/mean": -9.687859153747558, "rewards/ADERawReward/std": 6.932486724853516, "rewards/StrictFormatReward/mean": 0.41562500298023225, "rewards/StrictFormatReward/std": 2.435987985134125, "sampling/importance_sampling_ratio/max": 2.7624801874160765, "sampling/importance_sampling_ratio/mean": 0.39736475646495817, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.469653701782226, "sampling/sampling_logp_difference/mean": 0.025803259573876858, "step": 140, "step_time": 30.751169363802184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0067708335351198915, "completions/max_length": 996.8, "completions/max_terminated_length": 927.2, "completions/mean_length": 743.0797119140625, "completions/mean_terminated_length": 741.1633605957031, "completions/min_length": 364.4, "completions/min_terminated_length": 364.4, "entropy": 0.771180788675944, "epoch": 0.02571575518601063, "frac_reward_zero_std": 0.0, "grad_norm": 0.322265625, "learning_rate": 1e-06, "loss": -0.0135, "num_tokens": 67171268.0, "reward": -9.683474731445312, "reward_std": 6.344115209579468, "rewards/ADERawReward/mean": -9.721026802062989, "rewards/ADERawReward/std": 6.331903743743896, "rewards/StrictFormatReward/mean": 0.3755208313465118, "rewards/StrictFormatReward/std": 2.536852979660034, "sampling/importance_sampling_ratio/max": 2.6906844854354857, "sampling/importance_sampling_ratio/mean": 0.3939074516296387, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.591675233840943, "sampling/sampling_logp_difference/mean": 0.026031140238046646, "step": 150, "step_time": 30.759532438102177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.009375000186264516, "completions/max_length": 1001.1, "completions/max_terminated_length": 928.5, "completions/mean_length": 743.9906616210938, "completions/mean_terminated_length": 741.3461547851563, "completions/min_length": 396.0, "completions/min_terminated_length": 396.0, "entropy": 0.772610588868459, "epoch": 0.027430138865078006, "frac_reward_zero_std": 0.0, "grad_norm": 0.291015625, "learning_rate": 1e-06, "loss": -0.006, "num_tokens": 71648274.0, "reward": -9.378812217712403, "reward_std": 6.656644868850708, "rewards/ADERawReward/mean": -9.419801712036133, "rewards/ADERawReward/std": 6.638844442367554, "rewards/StrictFormatReward/mean": 0.4098958343267441, "rewards/StrictFormatReward/std": 2.457722854614258, "sampling/importance_sampling_ratio/max": 2.8564789295196533, "sampling/importance_sampling_ratio/mean": 0.3868180692195892, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.245336532592773, "sampling/sampling_logp_difference/mean": 0.026049494743347168, "step": 160, "step_time": 31.00511598830053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00572916679084301, "completions/max_length": 990.5, "completions/max_terminated_length": 943.8, "completions/mean_length": 742.833349609375, "completions/mean_terminated_length": 741.2216247558594, "completions/min_length": 507.2, "completions/min_terminated_length": 507.2, "entropy": 0.7669690032800038, "epoch": 0.02914452254414538, "frac_reward_zero_std": 0.0, "grad_norm": 0.2421875, "learning_rate": 1e-06, "loss": 0.0046, "num_tokens": 76122162.0, "reward": -9.605478668212891, "reward_std": 5.803140306472779, "rewards/ADERawReward/mean": -9.645895195007324, "rewards/ADERawReward/std": 5.7928056716918945, "rewards/StrictFormatReward/mean": 0.4041666634380817, "rewards/StrictFormatReward/std": 2.4726577758789063, "sampling/importance_sampling_ratio/max": 2.71953125, "sampling/importance_sampling_ratio/mean": 0.4068588227033615, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.668109703063965, "sampling/sampling_logp_difference/mean": 0.025866417028009892, "step": 170, "step_time": 30.801190392104036 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0067708335351198915, "completions/max_length": 992.8, "completions/max_terminated_length": 902.9, "completions/mean_length": 745.7635559082031, "completions/mean_terminated_length": 743.8595642089844, "completions/min_length": 539.9, "completions/min_terminated_length": 539.9, "entropy": 0.7681126793225607, "epoch": 0.030858906223212756, "frac_reward_zero_std": 0.0, "grad_norm": 0.330078125, "learning_rate": 1e-06, "loss": -0.0009, "num_tokens": 80601532.0, "reward": -10.118904399871827, "reward_std": 6.3121555805206295, "rewards/ADERawReward/mean": -10.168487930297852, "rewards/ADERawReward/std": 6.31596007347107, "rewards/StrictFormatReward/mean": 0.49583332538604735, "rewards/StrictFormatReward/std": 2.2922281503677366, "sampling/importance_sampling_ratio/max": 2.808049750328064, "sampling/importance_sampling_ratio/mean": 0.40117439031600954, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.4242959260940555, "sampling/sampling_logp_difference/mean": 0.026261823065578938, "step": 180, "step_time": 30.95573297930241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.009375000186264516, "completions/max_length": 1006.3, "completions/max_terminated_length": 940.1, "completions/mean_length": 745.2932495117187, "completions/mean_terminated_length": 742.6480163574219, "completions/min_length": 493.3, "completions/min_terminated_length": 493.3, "entropy": 0.7715065141518911, "epoch": 0.03257328990228013, "frac_reward_zero_std": 0.0, "grad_norm": 0.349609375, "learning_rate": 1e-06, "loss": -0.0167, "num_tokens": 85080239.0, "reward": -9.032559633255005, "reward_std": 5.6337450504302975, "rewards/ADERawReward/mean": -9.070111560821534, "rewards/ADERawReward/std": 5.615639925003052, "rewards/StrictFormatReward/mean": 0.3755208343267441, "rewards/StrictFormatReward/std": 2.532916522026062, "sampling/importance_sampling_ratio/max": 2.906028723716736, "sampling/importance_sampling_ratio/mean": 0.43047145903110506, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.560648036003113, "sampling/sampling_logp_difference/mean": 0.02589159458875656, "step": 190, "step_time": 31.04905029379588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.004687500139698386, "completions/max_length": 967.5, "completions/max_terminated_length": 901.5, "completions/mean_length": 743.9588745117187, "completions/mean_terminated_length": 742.6530639648438, "completions/min_length": 482.7, "completions/min_terminated_length": 482.7, "entropy": 0.7573511083920796, "epoch": 0.0342876735813475, "frac_reward_zero_std": 0.0, "grad_norm": 0.373046875, "learning_rate": 1e-06, "loss": -0.0071, "num_tokens": 89554992.0, "reward": -9.594873905181885, "reward_std": 7.4608996391296385, "rewards/ADERawReward/mean": -9.645030212402343, "rewards/ADERawReward/std": 7.4621100425720215, "rewards/StrictFormatReward/mean": 0.5015625014901162, "rewards/StrictFormatReward/std": 2.2230212569236754, "sampling/importance_sampling_ratio/max": 2.762194800376892, "sampling/importance_sampling_ratio/mean": 0.3942416876554489, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.715522122383118, "sampling/sampling_logp_difference/mean": 0.02548809293657541, "step": 200, "step_time": 30.452962850301994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.008854166883975268, "completions/max_length": 1002.2, "completions/max_terminated_length": 908.9, "completions/mean_length": 749.1671997070313, "completions/mean_terminated_length": 746.7061828613281, "completions/min_length": 479.3, "completions/min_terminated_length": 479.3, "entropy": 0.7769242246945699, "epoch": 0.03600205726041488, "frac_reward_zero_std": 0.0, "grad_norm": 0.349609375, "learning_rate": 1e-06, "loss": -0.0264, "num_tokens": 94041409.0, "reward": -9.6312180519104, "reward_std": 5.488570928573608, "rewards/ADERawReward/mean": -9.677363443374634, "rewards/ADERawReward/std": 5.482590246200561, "rewards/StrictFormatReward/mean": 0.46145834028720856, "rewards/StrictFormatReward/std": 2.369260883331299, "sampling/importance_sampling_ratio/max": 2.827503228187561, "sampling/importance_sampling_ratio/mean": 0.39889043271541597, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.615737795829773, "sampling/sampling_logp_difference/mean": 0.02642081268131733, "step": 210, "step_time": 31.002005700898007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.007291666883975268, "completions/max_length": 1011.2, "completions/max_terminated_length": 917.8, "completions/mean_length": 744.7901306152344, "completions/mean_terminated_length": 742.7363525390625, "completions/min_length": 326.2, "completions/min_terminated_length": 326.2, "entropy": 0.7611714641253153, "epoch": 0.03771644093948225, "frac_reward_zero_std": 0.0, "grad_norm": 0.2470703125, "learning_rate": 1e-06, "loss": -0.0072, "num_tokens": 98520158.0, "reward": -9.730458450317382, "reward_std": 6.581094980239868, "rewards/ADERawReward/mean": -9.778323078155518, "rewards/ADERawReward/std": 6.570281600952148, "rewards/StrictFormatReward/mean": 0.478645833581686, "rewards/StrictFormatReward/std": 2.319968819618225, "sampling/importance_sampling_ratio/max": 2.9045305013656617, "sampling/importance_sampling_ratio/mean": 0.4044792056083679, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.901871252059936, "sampling/sampling_logp_difference/mean": 0.026320008002221583, "step": 220, "step_time": 31.057802741799968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00833333358168602, "completions/max_length": 1006.6, "completions/max_terminated_length": 907.8, "completions/mean_length": 744.116162109375, "completions/mean_terminated_length": 741.7729553222656, "completions/min_length": 364.6, "completions/min_terminated_length": 364.6, "entropy": 0.7634056508541107, "epoch": 0.03943082461854963, "frac_reward_zero_std": 0.0, "grad_norm": 0.35546875, "learning_rate": 1e-06, "loss": 0.0058, "num_tokens": 102995469.0, "reward": -9.594787311553954, "reward_std": 6.4190247535705565, "rewards/ADERawReward/mean": -9.632339668273925, "rewards/ADERawReward/std": 6.41018123626709, "rewards/StrictFormatReward/mean": 0.37552083283662796, "rewards/StrictFormatReward/std": 2.4954795360565187, "sampling/importance_sampling_ratio/max": 2.7927833795547485, "sampling/importance_sampling_ratio/mean": 0.38037638664245604, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.303525161743164, "sampling/sampling_logp_difference/mean": 0.026079374738037585, "step": 230, "step_time": 30.925862592204066 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015625000232830644, "completions/max_length": 1024.0, "completions/max_terminated_length": 922.7, "completions/mean_length": 747.7349060058593, "completions/mean_terminated_length": 743.3495300292968, "completions/min_length": 425.3, "completions/min_terminated_length": 425.3, "entropy": 0.7614292343457539, "epoch": 0.04114520829761701, "frac_reward_zero_std": 0.0, "grad_norm": 0.3984375, "learning_rate": 1e-06, "loss": 0.0014, "num_tokens": 107478240.0, "reward": -9.159432983398437, "reward_std": 5.858185243606568, "rewards/ADERawReward/mean": -9.192401695251466, "rewards/ADERawReward/std": 5.8467125415802, "rewards/StrictFormatReward/mean": 0.3296875016763806, "rewards/StrictFormatReward/std": 2.6086880922317506, "sampling/importance_sampling_ratio/max": 2.8600486755371093, "sampling/importance_sampling_ratio/mean": 0.38983087837696073, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.5773406505584715, "sampling/sampling_logp_difference/mean": 0.026277855411171912, "step": 240, "step_time": 31.37704919550306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.011458333488553763, "completions/max_length": 1012.8, "completions/max_terminated_length": 905.7, "completions/mean_length": 743.3661682128907, "completions/mean_terminated_length": 740.1103088378907, "completions/min_length": 399.4, "completions/min_terminated_length": 399.4, "entropy": 0.753006378809611, "epoch": 0.04285959197668438, "frac_reward_zero_std": 0.0, "grad_norm": 0.35546875, "learning_rate": 1e-06, "loss": -0.0194, "num_tokens": 111952815.0, "reward": -9.348959684371948, "reward_std": 5.709445333480835, "rewards/ADERawReward/mean": -9.387657499313354, "rewards/ADERawReward/std": 5.683103704452515, "rewards/StrictFormatReward/mean": 0.38697916716337205, "rewards/StrictFormatReward/std": 2.5156437873840334, "sampling/importance_sampling_ratio/max": 2.7942948818206785, "sampling/importance_sampling_ratio/mean": 0.3898850232362747, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.9041135787963865, "sampling/sampling_logp_difference/mean": 0.026003769040107726, "step": 250, "step_time": 31.078463642198766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016145833814516664, "completions/max_length": 1017.2, "completions/max_terminated_length": 912.4, "completions/mean_length": 749.7567993164063, "completions/mean_terminated_length": 745.2748840332031, "completions/min_length": 587.3, "completions/min_terminated_length": 587.3, "entropy": 0.7615062018235524, "epoch": 0.04457397565575176, "frac_reward_zero_std": 0.0, "grad_norm": 0.3984375, "learning_rate": 1e-06, "loss": -0.0051, "num_tokens": 116440044.0, "reward": -9.92240343093872, "reward_std": 5.772085857391358, "rewards/ADERawReward/mean": -9.958809566497802, "rewards/ADERawReward/std": 5.756062030792236, "rewards/StrictFormatReward/mean": 0.36406250670552254, "rewards/StrictFormatReward/std": 2.54167058467865, "sampling/importance_sampling_ratio/max": 2.7225013256072996, "sampling/importance_sampling_ratio/mean": 0.3842493683099747, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.781629824638367, "sampling/sampling_logp_difference/mean": 0.02624006476253271, "step": 260, "step_time": 31.103271031705663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.008333333535119891, "completions/max_length": 998.2, "completions/max_terminated_length": 911.1, "completions/mean_length": 743.7968994140625, "completions/mean_terminated_length": 741.4540466308594, "completions/min_length": 510.3, "completions/min_terminated_length": 510.3, "entropy": 0.7562108953793844, "epoch": 0.04628835933481913, "frac_reward_zero_std": 0.0, "grad_norm": 0.1953125, "learning_rate": 1e-06, "loss": -0.0247, "num_tokens": 120916390.0, "reward": -9.01723370552063, "reward_std": 5.40621428489685, "rewards/ADERawReward/mean": -9.065098094940186, "rewards/ADERawReward/std": 5.384741115570068, "rewards/StrictFormatReward/mean": 0.4786458298563957, "rewards/StrictFormatReward/std": 2.2954741954803466, "sampling/importance_sampling_ratio/max": 2.789908194541931, "sampling/importance_sampling_ratio/mean": 0.40749328434467313, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.034530830383301, "sampling/sampling_logp_difference/mean": 0.026305202022194864, "step": 270, "step_time": 30.820387327201026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.009375000139698386, "completions/max_length": 1011.8, "completions/max_terminated_length": 914.4, "completions/mean_length": 745.3156433105469, "completions/mean_terminated_length": 742.68203125, "completions/min_length": 492.1, "completions/min_terminated_length": 492.1, "entropy": 0.757685911655426, "epoch": 0.04800274301388651, "frac_reward_zero_std": 0.0, "grad_norm": 0.349609375, "learning_rate": 1e-06, "loss": 0.002, "num_tokens": 125395556.0, "reward": -9.0065682888031, "reward_std": 5.3750749111175535, "rewards/ADERawReward/mean": -9.06875557899475, "rewards/ADERawReward/std": 5.365862655639648, "rewards/StrictFormatReward/mean": 0.6218749970197678, "rewards/StrictFormatReward/std": 1.9849600195884705, "sampling/importance_sampling_ratio/max": 2.791543960571289, "sampling/importance_sampling_ratio/mean": 0.36947061419487, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.032503914833069, "sampling/sampling_logp_difference/mean": 0.02653008736670017, "step": 280, "step_time": 31.023602066392776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.009375000139698386, "completions/max_length": 1011.5, "completions/max_terminated_length": 906.5, "completions/mean_length": 745.1588745117188, "completions/mean_terminated_length": 742.5252624511719, "completions/min_length": 384.9, "completions/min_terminated_length": 384.9, "entropy": 0.7569859484831493, "epoch": 0.04971712669295388, "frac_reward_zero_std": 0.0, "grad_norm": 0.275390625, "learning_rate": 1e-06, "loss": -0.0025, "num_tokens": 129874581.0, "reward": -9.8903018951416, "reward_std": 5.79907751083374, "rewards/ADERawReward/mean": -9.936447715759277, "rewards/ADERawReward/std": 5.782765436172485, "rewards/StrictFormatReward/mean": 0.4614583358168602, "rewards/StrictFormatReward/std": 2.3571563005447387, "sampling/importance_sampling_ratio/max": 2.780865716934204, "sampling/importance_sampling_ratio/mean": 0.3752612203359604, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.010831832885742, "sampling/sampling_logp_difference/mean": 0.026486290618777275, "step": 290, "step_time": 31.036961890700333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.010937500186264515, "completions/max_length": 1024.0, "completions/max_terminated_length": 909.3, "completions/mean_length": 743.6302307128906, "completions/mean_terminated_length": 740.5345153808594, "completions/min_length": 545.1, "completions/min_terminated_length": 545.1, "entropy": 0.7479812840620677, "epoch": 0.05143151037202126, "frac_reward_zero_std": 0.0, "grad_norm": 0.296875, "learning_rate": 1e-06, "loss": 0.0075, "num_tokens": 134349103.0, "reward": -8.726549053192139, "reward_std": 5.281997394561768, "rewards/ADERawReward/mean": -8.780142164230346, "rewards/ADERawReward/std": 5.273246812820434, "rewards/StrictFormatReward/mean": 0.5359374985098839, "rewards/StrictFormatReward/std": 2.176564943790436, "sampling/importance_sampling_ratio/max": 2.9104627132415772, "sampling/importance_sampling_ratio/mean": 0.40789560675621034, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.580249309539795, "sampling/sampling_logp_difference/mean": 0.025899984315037726, "step": 300, "step_time": 31.32498294780089 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.007812500186264514, "completions/max_length": 998.0, "completions/max_terminated_length": 917.8, "completions/mean_length": 745.6083557128907, "completions/mean_terminated_length": 743.4208374023438, "completions/min_length": 550.1, "completions/min_terminated_length": 550.1, "entropy": 0.7465774556001027, "epoch": 0.05314589405108863, "frac_reward_zero_std": 0.0, "grad_norm": 0.353515625, "learning_rate": 1e-06, "loss": -0.0123, "num_tokens": 138828591.0, "reward": -9.077164840698241, "reward_std": 5.992494773864746, "rewards/ADERawReward/mean": -9.131331491470338, "rewards/ADERawReward/std": 5.984670972824096, "rewards/StrictFormatReward/mean": 0.5416666656732559, "rewards/StrictFormatReward/std": 2.169144892692566, "sampling/importance_sampling_ratio/max": 2.749341917037964, "sampling/importance_sampling_ratio/mean": 0.3683441460132599, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.0134562969207765, "sampling/sampling_logp_difference/mean": 0.026201115176081657, "step": 310, "step_time": 30.797605072900478 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.016145833814516664, "completions/max_length": 1024.0, "completions/max_terminated_length": 899.7, "completions/mean_length": 747.6823181152344, "completions/mean_terminated_length": 743.1548828125, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "entropy": 0.7556955059369405, "epoch": 0.05486027773015601, "frac_reward_zero_std": 0.0, "grad_norm": 0.3671875, "learning_rate": 1e-06, "loss": 0.006, "num_tokens": 143312285.0, "reward": -9.226322269439697, "reward_std": 6.171311950683593, "rewards/ADERawReward/mean": -9.265593338012696, "rewards/ADERawReward/std": 6.160300636291504, "rewards/StrictFormatReward/mean": 0.39270834550261496, "rewards/StrictFormatReward/std": 2.4976075172424315, "sampling/importance_sampling_ratio/max": 2.7495847940444946, "sampling/importance_sampling_ratio/mean": 0.384751632809639, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.004838466644287, "sampling/sampling_logp_difference/mean": 0.026612747460603714, "step": 320, "step_time": 31.281772803403147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.007812500093132257, "completions/max_length": 1001.5, "completions/max_terminated_length": 901.0, "completions/mean_length": 745.114599609375, "completions/mean_terminated_length": 742.9226379394531, "completions/min_length": 415.5, "completions/min_terminated_length": 415.5, "entropy": 0.7487649619579315, "epoch": 0.05657466140922338, "frac_reward_zero_std": 0.0, "grad_norm": 0.2099609375, "learning_rate": 1e-06, "loss": 0.0105, "num_tokens": 147791241.0, "reward": -8.852140951156617, "reward_std": 6.015402030944824, "rewards/ADERawReward/mean": -8.904588794708252, "rewards/ADERawReward/std": 6.00461745262146, "rewards/StrictFormatReward/mean": 0.5244791731238365, "rewards/StrictFormatReward/std": 2.198140048980713, "sampling/importance_sampling_ratio/max": 2.8983518362045286, "sampling/importance_sampling_ratio/mean": 0.38104148209095, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.405174565315247, "sampling/sampling_logp_difference/mean": 0.026203346997499467, "step": 330, "step_time": 30.943252457803464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.013541666883975267, "completions/max_length": 1024.0, "completions/max_terminated_length": 910.4, "completions/mean_length": 746.2218994140625, "completions/mean_terminated_length": 742.4164428710938, "completions/min_length": 451.8, "completions/min_terminated_length": 451.8, "entropy": 0.7482379992802938, "epoch": 0.05828904508829076, "frac_reward_zero_std": 0.0, "grad_norm": 0.2490234375, "learning_rate": 1e-06, "loss": -0.0044, "num_tokens": 152271555.0, "reward": -9.297206592559814, "reward_std": 5.522917318344116, "rewards/ADERawReward/mean": -9.338768911361694, "rewards/ADERawReward/std": 5.507352113723755, "rewards/StrictFormatReward/mean": 0.4156250014901161, "rewards/StrictFormatReward/std": 2.437586045265198, "sampling/importance_sampling_ratio/max": 2.782149076461792, "sampling/importance_sampling_ratio/mean": 0.37727462947368623, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.424329519271851, "sampling/sampling_logp_difference/mean": 0.0264679953455925, "step": 340, "step_time": 31.394037494297663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.012500000232830644, "completions/max_length": 1024.0, "completions/max_terminated_length": 929.2, "completions/mean_length": 747.1989868164062, "completions/mean_terminated_length": 743.6892578125, "completions/min_length": 461.7, "completions/min_terminated_length": 461.7, "entropy": 0.7526971201101939, "epoch": 0.06000342876735813, "frac_reward_zero_std": 0.0, "grad_norm": 0.373046875, "learning_rate": 1e-06, "loss": -0.0279, "num_tokens": 156753761.0, "reward": -8.87293882369995, "reward_std": 5.407143402099609, "rewards/ADERawReward/mean": -8.912782192230225, "rewards/ADERawReward/std": 5.394438552856445, "rewards/StrictFormatReward/mean": 0.3984375029802322, "rewards/StrictFormatReward/std": 2.4990755558013915, "sampling/importance_sampling_ratio/max": 2.7867547035217286, "sampling/importance_sampling_ratio/mean": 0.3730757534503937, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.122068333625793, "sampling/sampling_logp_difference/mean": 0.02654006313532591, "step": 350, "step_time": 31.515721038298217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.012500000325962901, "completions/max_length": 1007.6, "completions/max_terminated_length": 894.1, "completions/mean_length": 747.2932495117187, "completions/mean_terminated_length": 743.7852661132813, "completions/min_length": 591.8, "completions/min_terminated_length": 591.8, "entropy": 0.7486960550149282, "epoch": 0.06171781244642551, "frac_reward_zero_std": 0.0, "grad_norm": 0.2373046875, "learning_rate": 1e-06, "loss": -0.0224, "num_tokens": 161235908.0, "reward": -9.346630477905274, "reward_std": 5.943923234939575, "rewards/ADERawReward/mean": -9.397359657287598, "rewards/ADERawReward/std": 5.933114051818848, "rewards/StrictFormatReward/mean": 0.5072916746139526, "rewards/StrictFormatReward/std": 2.258626651763916, "sampling/importance_sampling_ratio/max": 2.783932328224182, "sampling/importance_sampling_ratio/mean": 0.3899604082107544, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.8976672172546385, "sampling/sampling_logp_difference/mean": 0.02663377095013857, "step": 360, "step_time": 31.072972166599357 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.010937500232830643, "completions/max_length": 1024.0, "completions/max_terminated_length": 907.6, "completions/mean_length": 743.9771057128906, "completions/mean_terminated_length": 740.8768371582031, "completions/min_length": 373.6, "completions/min_terminated_length": 373.6, "entropy": 0.7463085273901622, "epoch": 0.06343219612549289, "frac_reward_zero_std": 0.0, "grad_norm": 0.43359375, "learning_rate": 1e-06, "loss": -0.0195, "num_tokens": 165711768.0, "reward": -9.58823890686035, "reward_std": 7.584380769729615, "rewards/ADERawReward/mean": -9.62750973701477, "rewards/ADERawReward/std": 7.566325521469116, "rewards/StrictFormatReward/mean": 0.3927083402872086, "rewards/StrictFormatReward/std": 2.502521538734436, "sampling/importance_sampling_ratio/max": 2.824156880378723, "sampling/importance_sampling_ratio/mean": 0.38374905586242675, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.386010980606079, "sampling/sampling_logp_difference/mean": 0.02665374930948019, "step": 370, "step_time": 31.397882584496983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.009895833674818277, "completions/max_length": 1000.8, "completions/max_terminated_length": 913.6, "completions/mean_length": 746.1416870117188, "completions/mean_terminated_length": 743.3394592285156, "completions/min_length": 440.4, "completions/min_terminated_length": 440.4, "entropy": 0.7471979439258576, "epoch": 0.06514657980456026, "frac_reward_zero_std": 0.0, "grad_norm": 0.33203125, "learning_rate": 1e-06, "loss": 0.0032, "num_tokens": 170192072.0, "reward": -9.293725442886352, "reward_std": 5.743723678588867, "rewards/ADERawReward/mean": -9.341016817092896, "rewards/ADERawReward/std": 5.729398393630982, "rewards/StrictFormatReward/mean": 0.4729166701436043, "rewards/StrictFormatReward/std": 2.320163094997406, "sampling/importance_sampling_ratio/max": 2.869658589363098, "sampling/importance_sampling_ratio/mean": 0.3773952782154083, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.138100743293762, "sampling/sampling_logp_difference/mean": 0.026742291823029517, "step": 380, "step_time": 30.96988229679846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.012500000325962901, "completions/max_length": 1016.4, "completions/max_terminated_length": 914.7, "completions/mean_length": 748.4885559082031, "completions/mean_terminated_length": 744.993017578125, "completions/min_length": 477.5, "completions/min_terminated_length": 477.5, "entropy": 0.7559548437595367, "epoch": 0.06686096348362763, "frac_reward_zero_std": 0.0, "grad_norm": 0.24609375, "learning_rate": 1e-06, "loss": -0.0181, "num_tokens": 174677202.0, "reward": -9.28088207244873, "reward_std": 5.592328834533691, "rewards/ADERawReward/mean": -9.324163150787353, "rewards/ADERawReward/std": 5.577477645874024, "rewards/StrictFormatReward/mean": 0.43281249795109034, "rewards/StrictFormatReward/std": 2.4112179040908814, "sampling/importance_sampling_ratio/max": 2.825889301300049, "sampling/importance_sampling_ratio/mean": 0.3906464457511902, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.902947115898132, "sampling/sampling_logp_difference/mean": 0.026762423664331438, "step": 390, "step_time": 31.20840191419411 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.007812500186264514, "completions/max_length": 992.4, "completions/max_terminated_length": 934.3, "completions/mean_length": 745.6031433105469, "completions/mean_terminated_length": 743.4128295898438, "completions/min_length": 437.2, "completions/min_terminated_length": 437.2, "entropy": 0.7531329174836476, "epoch": 0.068575347162695, "frac_reward_zero_std": 0.0, "grad_norm": 0.376953125, "learning_rate": 1e-06, "loss": 0.008, "num_tokens": 179156696.0, "reward": -9.488264322280884, "reward_std": 5.980498790740967, "rewards/ADERawReward/mean": -9.5326913356781, "rewards/ADERawReward/std": 5.96862416267395, "rewards/StrictFormatReward/mean": 0.44427084028720853, "rewards/StrictFormatReward/std": 2.4021025896072388, "sampling/importance_sampling_ratio/max": 2.796902632713318, "sampling/importance_sampling_ratio/mean": 0.38265294432640073, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.037104201316834, "sampling/sampling_logp_difference/mean": 0.026842619478702544, "step": 400, "step_time": 30.975298491402647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.008854166977107525, "completions/max_length": 1020.0, "completions/max_terminated_length": 935.6, "completions/mean_length": 746.836474609375, "completions/mean_terminated_length": 744.3670227050782, "completions/min_length": 503.7, "completions/min_terminated_length": 503.7, "entropy": 0.7571933090686798, "epoch": 0.07028973084176239, "frac_reward_zero_std": 0.0, "grad_norm": 0.30859375, "learning_rate": 1e-06, "loss": -0.0061, "num_tokens": 183639214.0, "reward": -8.974328899383545, "reward_std": 6.1143780708312985, "rewards/ADERawReward/mean": -9.019901895523072, "rewards/ADERawReward/std": 6.102041292190552, "rewards/StrictFormatReward/mean": 0.45572916865348817, "rewards/StrictFormatReward/std": 2.3774968147277833, "sampling/importance_sampling_ratio/max": 2.655148983001709, "sampling/importance_sampling_ratio/mean": 0.36126451194286346, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.974502754211426, "sampling/sampling_logp_difference/mean": 0.027187051996588708, "step": 410, "step_time": 31.259745221505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.010416666930541395, "completions/max_length": 1017.3, "completions/max_terminated_length": 918.2, "completions/mean_length": 747.1531433105469, "completions/mean_terminated_length": 744.2319213867188, "completions/min_length": 565.9, "completions/min_terminated_length": 565.9, "entropy": 0.7566624025503794, "epoch": 0.07200411452082976, "frac_reward_zero_std": 0.0, "grad_norm": 0.3359375, "learning_rate": 1e-06, "loss": -0.0087, "num_tokens": 188121876.0, "reward": -8.923426342010497, "reward_std": 6.084955930709839, "rewards/ADERawReward/mean": -8.9724365234375, "rewards/ADERawReward/std": 6.07550630569458, "rewards/StrictFormatReward/mean": 0.4901041708886623, "rewards/StrictFormatReward/std": 2.2838862776756286, "sampling/importance_sampling_ratio/max": 2.812648606300354, "sampling/importance_sampling_ratio/mean": 0.3750308662652969, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.534145450592041, "sampling/sampling_logp_difference/mean": 0.026898518949747086, "step": 420, "step_time": 31.253269567198004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.010416666837409138, "completions/max_length": 1000.9, "completions/max_terminated_length": 899.6, "completions/mean_length": 744.9640930175781, "completions/mean_terminated_length": 742.0363525390625, "completions/min_length": 414.5, "completions/min_terminated_length": 414.5, "entropy": 0.7489292581876119, "epoch": 0.07371849819989713, "frac_reward_zero_std": 0.0, "grad_norm": 0.58203125, "learning_rate": 1e-06, "loss": -0.0073, "num_tokens": 192599695.0, "reward": -9.504335880279541, "reward_std": 5.772246503829956, "rewards/ADERawReward/mean": -9.548762798309326, "rewards/ADERawReward/std": 5.758863925933838, "rewards/StrictFormatReward/mean": 0.4442708283662796, "rewards/StrictFormatReward/std": 2.37346408367157, "sampling/importance_sampling_ratio/max": 2.820726418495178, "sampling/importance_sampling_ratio/mean": 0.36917952001094817, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.74177827835083, "sampling/sampling_logp_difference/mean": 0.026925336755812167, "step": 430, "step_time": 31.05574051309668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.013541666977107524, "completions/max_length": 1003.0, "completions/max_terminated_length": 914.5, "completions/mean_length": 746.8135620117188, "completions/mean_terminated_length": 743.0150817871094, "completions/min_length": 349.1, "completions/min_terminated_length": 349.1, "entropy": 0.7525838236014049, "epoch": 0.0754328818789645, "frac_reward_zero_std": 0.0, "grad_norm": 0.3359375, "learning_rate": 1e-06, "loss": -0.0216, "num_tokens": 197081145.0, "reward": -8.942778491973877, "reward_std": 5.134439134597779, "rewards/ADERawReward/mean": -8.988924121856689, "rewards/ADERawReward/std": 5.119956731796265, "rewards/StrictFormatReward/mean": 0.4614583387970924, "rewards/StrictFormatReward/std": 2.3441874384880066, "sampling/importance_sampling_ratio/max": 2.7575156688690186, "sampling/importance_sampling_ratio/mean": 0.3831501841545105, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.117226243019104, "sampling/sampling_logp_difference/mean": 0.02673897407948971, "step": 440, "step_time": 31.136345591007558 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00989583358168602, "completions/max_length": 1024.0, "completions/max_terminated_length": 962.1, "completions/mean_length": 746.3265747070312, "completions/mean_terminated_length": 743.5626037597656, "completions/min_length": 502.5, "completions/min_terminated_length": 502.5, "entropy": 0.7494863112767537, "epoch": 0.07714726555803189, "frac_reward_zero_std": 0.0, "grad_norm": 0.396484375, "learning_rate": 1e-06, "loss": 0.0015, "num_tokens": 201562060.0, "reward": -8.833820581436157, "reward_std": 6.585151100158692, "rewards/ADERawReward/mean": -8.883976650238036, "rewards/ADERawReward/std": 6.576332855224609, "rewards/StrictFormatReward/mean": 0.5015625029802322, "rewards/StrictFormatReward/std": 2.246139848232269, "sampling/importance_sampling_ratio/max": 2.858155703544617, "sampling/importance_sampling_ratio/mean": 0.38457230627536776, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.555598568916321, "sampling/sampling_logp_difference/mean": 0.026770622283220292, "step": 450, "step_time": 31.314683586404136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.010937500325962902, "completions/max_length": 1012.7, "completions/max_terminated_length": 908.7, "completions/mean_length": 747.1239807128907, "completions/mean_terminated_length": 744.033935546875, "completions/min_length": 452.2, "completions/min_terminated_length": 452.2, "entropy": 0.7497527619202932, "epoch": 0.07886164923709926, "frac_reward_zero_std": 0.0, "grad_norm": 0.2392578125, "learning_rate": 1e-06, "loss": -0.0312, "num_tokens": 206044186.0, "reward": -8.526823663711548, "reward_std": 4.98410964012146, "rewards/ADERawReward/mean": -8.568385744094849, "rewards/ADERawReward/std": 4.961003923416138, "rewards/StrictFormatReward/mean": 0.415625, "rewards/StrictFormatReward/std": 2.4348717212677, "sampling/importance_sampling_ratio/max": 2.7420751094818114, "sampling/importance_sampling_ratio/mean": 0.3720589101314545, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.188339471817017, "sampling/sampling_logp_difference/mean": 0.026370251737535, "step": 460, "step_time": 31.211517236700455 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.010416666837409138, "completions/max_length": 1013.2, "completions/max_terminated_length": 913.8, "completions/mean_length": 747.784912109375, "completions/mean_terminated_length": 744.8851257324219, "completions/min_length": 416.8, "completions/min_terminated_length": 416.8, "entropy": 0.7524498959382375, "epoch": 0.08057603291616663, "frac_reward_zero_std": 0.0, "grad_norm": 0.349609375, "learning_rate": 1e-06, "loss": -0.0088, "num_tokens": 210527581.0, "reward": -9.329790830612183, "reward_std": 5.533093023300171, "rewards/ADERawReward/mean": -9.378228092193604, "rewards/ADERawReward/std": 5.5270590543746945, "rewards/StrictFormatReward/mean": 0.4843750029802322, "rewards/StrictFormatReward/std": 2.30391206741333, "sampling/importance_sampling_ratio/max": 2.7844461679458616, "sampling/importance_sampling_ratio/mean": 0.36996753215789796, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.251289129257202, "sampling/sampling_logp_difference/mean": 0.026920023374259472, "step": 470, "step_time": 31.291990611601797 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.008333333535119891, "completions/max_length": 1006.7, "completions/max_terminated_length": 924.5, "completions/mean_length": 744.3693054199218, "completions/mean_terminated_length": 742.0288696289062, "completions/min_length": 345.2, "completions/min_terminated_length": 345.2, "entropy": 0.7420653978983561, "epoch": 0.08229041659523402, "frac_reward_zero_std": 0.0, "grad_norm": 0.25390625, "learning_rate": 1e-06, "loss": -0.0135, "num_tokens": 215005026.0, "reward": -8.811880826950073, "reward_std": 5.741937494277954, "rewards/ADERawReward/mean": -8.863182735443115, "rewards/ADERawReward/std": 5.721714019775391, "rewards/StrictFormatReward/mean": 0.5130208387970925, "rewards/StrictFormatReward/std": 2.2228100419044496, "sampling/importance_sampling_ratio/max": 2.887178134918213, "sampling/importance_sampling_ratio/mean": 0.3799948424100876, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.423047304153442, "sampling/sampling_logp_difference/mean": 0.026753966324031354, "step": 480, "step_time": 31.235763939598108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.007812500232830644, "completions/max_length": 1007.4, "completions/max_terminated_length": 921.0, "completions/mean_length": 745.5041809082031, "completions/mean_terminated_length": 743.31552734375, "completions/min_length": 503.1, "completions/min_terminated_length": 503.1, "entropy": 0.7548379202683767, "epoch": 0.08400480027430139, "frac_reward_zero_std": 0.0, "grad_norm": 0.302734375, "learning_rate": 1e-06, "loss": 0.0051, "num_tokens": 219484362.0, "reward": -8.726992988586426, "reward_std": 5.2785539627075195, "rewards/ADERawReward/mean": -8.783451509475707, "rewards/ADERawReward/std": 5.263514995574951, "rewards/StrictFormatReward/mean": 0.5645833373069763, "rewards/StrictFormatReward/std": 2.121866428852081, "sampling/importance_sampling_ratio/max": 2.8357564210891724, "sampling/importance_sampling_ratio/mean": 0.36869050562381744, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.648310852050781, "sampling/sampling_logp_difference/mean": 0.026755044236779213, "step": 490, "step_time": 31.2714247025011 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.010416666930541395, "completions/max_length": 1011.9, "completions/max_terminated_length": 899.3, "completions/mean_length": 745.266162109375, "completions/mean_terminated_length": 742.3389892578125, "completions/min_length": 504.1, "completions/min_terminated_length": 504.1, "entropy": 0.7439370552698771, "epoch": 0.08571918395336876, "frac_reward_zero_std": 0.0, "grad_norm": 0.337890625, "learning_rate": 1e-06, "loss": -0.0111, "num_tokens": 223962569.0, "reward": -8.893217849731446, "reward_std": 6.001188564300537, "rewards/ADERawReward/mean": -8.93993616104126, "rewards/ADERawReward/std": 5.9909703731536865, "rewards/StrictFormatReward/mean": 0.46718750298023226, "rewards/StrictFormatReward/std": 2.34447557926178, "sampling/importance_sampling_ratio/max": 2.847609305381775, "sampling/importance_sampling_ratio/mean": 0.4031797587871552, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.063726282119751, "sampling/sampling_logp_difference/mean": 0.02628465835005045, "step": 500, "step_time": 31.151053833100015 } ], "logging_steps": 10, "max_steps": 730, "num_input_tokens_seen": 223962569, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 16, "trial_name": null, "trial_params": null }