{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.4076086956521739, "eval_steps": 500, "global_step": 150, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1671.2799072265625, "completions/mean_terminated_length": 1398.4827880859375, "completions/min_length": 992.0, "completions/min_terminated_length": 992.0, "entropy": 0.21248915791511536, "epoch": 0.002717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.016694288700819016, "learning_rate": 0.0, "loss": 0.050320815294981, "num_tokens": 86304.0, "reward": -0.21605467796325684, "reward_std": 0.656523585319519, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.8160547018051147, "rewards/length_penalty/std": 0.18964654207229614, "sampling/importance_sampling_ratio/max": 1.5222556591033936, "sampling/importance_sampling_ratio/mean": 0.9927070140838623, "sampling/importance_sampling_ratio/min": 0.6435883045196533, "sampling/sampling_logp_difference/max": 0.44069600105285645, "sampling/sampling_logp_difference/mean": 0.013988969847559929, "step": 1, "step_time": 27.397584670223296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6800000071525574, "completions/max_length": 2048.0, "completions/max_terminated_length": 1991.0, "completions/mean_length": 1941.47998046875, "completions/mean_terminated_length": 1715.125, "completions/min_length": 1079.0, "completions/min_terminated_length": 1079.0, "entropy": 0.2905632793903351, "epoch": 0.005434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.017825370654463768, "learning_rate": 5e-06, "loss": 0.0580969899892807, "num_tokens": 186618.0, "reward": -0.7279882431030273, "reward_std": 0.4917677640914917, "rewards/correctness/mean": 0.2199999988079071, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.9479882717132568, "rewards/length_penalty/std": 0.10518474876880646, "sampling/importance_sampling_ratio/max": 1.450933814048767, "sampling/importance_sampling_ratio/mean": 0.9901063442230225, "sampling/importance_sampling_ratio/min": 0.3688630163669586, "sampling/sampling_logp_difference/max": 0.9973299503326416, "sampling/sampling_logp_difference/mean": 0.01780563034117222, "step": 2, "step_time": 29.082111555384472 }, { "clip_ratio/high_max": 0.0008598717977292836, "clip_ratio/high_mean": 0.0008598717977292836, "clip_ratio/low_mean": 0.00014835237379884347, "clip_ratio/low_min": 0.00014835237379884347, "clip_ratio/region_mean": 0.0010082241846248508, "completions/clipped_ratio": 0.4399999976158142, "completions/max_length": 2048.0, "completions/max_terminated_length": 2002.0, "completions/mean_length": 1746.3800048828125, "completions/mean_terminated_length": 1509.3929443359375, "completions/min_length": 760.0, "completions/min_terminated_length": 760.0, "entropy": 0.2670243740081787, "epoch": 0.008152173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.017832612618803978, "learning_rate": 1e-05, "loss": 0.07830622792243958, "num_tokens": 276207.0, "reward": -0.25272461771965027, "reward_std": 0.6300041675567627, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.8527246117591858, "rewards/length_penalty/std": 0.17827628552913666, "sampling/importance_sampling_ratio/max": 1.4769505262374878, "sampling/importance_sampling_ratio/mean": 0.9909622669219971, "sampling/importance_sampling_ratio/min": 0.5282294750213623, "sampling/sampling_logp_difference/max": 0.6382244825363159, "sampling/sampling_logp_difference/mean": 0.017083315178751945, "step": 3, "step_time": 27.76276267436333 }, { "clip_ratio/high_max": 0.0004195733112283051, "clip_ratio/high_mean": 0.0004195733112283051, "clip_ratio/low_mean": 6.936315767234192e-05, "clip_ratio/low_min": 6.936315767234192e-05, "clip_ratio/region_mean": 0.0004889364703558385, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1971.0, "completions/mean_length": 1479.6199951171875, "completions/mean_terminated_length": 1279.9189453125, "completions/min_length": 692.0, "completions/min_terminated_length": 692.0, "entropy": 0.24769674837589264, "epoch": 0.010869565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.017356887459754944, "learning_rate": 1.5e-05, "loss": 0.08260345458984375, "num_tokens": 353088.0, "reward": 0.01752929575741291, "reward_std": 0.6234192848205566, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.7224707007408142, "rewards/length_penalty/std": 0.21228434145450592, "sampling/importance_sampling_ratio/max": 1.4611026048660278, "sampling/importance_sampling_ratio/mean": 0.9913830161094666, "sampling/importance_sampling_ratio/min": 0.6426271200180054, "sampling/sampling_logp_difference/max": 0.44219064712524414, "sampling/sampling_logp_difference/mean": 0.015955109149217606, "step": 4, "step_time": 27.095191412372515 }, { "clip_ratio/high_max": 0.0009722063725348562, "clip_ratio/high_mean": 0.0009722063725348562, "clip_ratio/low_mean": 8.280473921331576e-05, "clip_ratio/low_min": 8.280473921331576e-05, "clip_ratio/region_mean": 0.0010550110950134695, "completions/clipped_ratio": 0.8199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 1668.0, "completions/mean_length": 1934.659912109375, "completions/mean_terminated_length": 1418.3333740234375, "completions/min_length": 1111.0, "completions/min_terminated_length": 1111.0, "entropy": 0.26989140510559084, "epoch": 0.01358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.016684621572494507, "learning_rate": 2e-05, "loss": 0.09316413104534149, "num_tokens": 453531.0, "reward": -0.7646582126617432, "reward_std": 0.508796215057373, "rewards/correctness/mean": 0.18000000715255737, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.9446582198143005, "rewards/length_penalty/std": 0.1250941902399063, "sampling/importance_sampling_ratio/max": 1.4654922485351562, "sampling/importance_sampling_ratio/mean": 0.9907313585281372, "sampling/importance_sampling_ratio/min": 0.4321030080318451, "sampling/sampling_logp_difference/max": 0.8390913009643555, "sampling/sampling_logp_difference/mean": 0.017205415293574333, "step": 5, "step_time": 29.35069163166918 }, { "clip_ratio/high_max": 0.0013098036171868444, "clip_ratio/high_mean": 0.0013098036171868444, "clip_ratio/low_mean": 8.449839442619123e-05, "clip_ratio/low_min": 8.449839442619123e-05, "clip_ratio/region_mean": 0.0013943019905127585, "completions/clipped_ratio": 0.5199999809265137, "completions/max_length": 2048.0, "completions/max_terminated_length": 2010.0, "completions/mean_length": 1694.8199462890625, "completions/mean_terminated_length": 1312.2083740234375, "completions/min_length": 815.0, "completions/min_terminated_length": 815.0, "entropy": 0.25111753940582277, "epoch": 0.016304347826086956, "frac_reward_zero_std": 0.0, "grad_norm": 0.017302578315138817, "learning_rate": 2.5e-05, "loss": 0.058736853301525116, "num_tokens": 541262.0, "reward": -0.3275488317012787, "reward_std": 0.692110538482666, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.8275488018989563, "rewards/length_penalty/std": 0.2189619243144989, "sampling/importance_sampling_ratio/max": 1.5202347040176392, "sampling/importance_sampling_ratio/mean": 0.9910738468170166, "sampling/importance_sampling_ratio/min": 0.5493231415748596, "sampling/sampling_logp_difference/max": 0.5990684032440186, "sampling/sampling_logp_difference/mean": 0.016461573541164398, "step": 6, "step_time": 27.52322673960589 }, { "clip_ratio/high_max": 0.001128643041010946, "clip_ratio/high_mean": 0.001128643041010946, "clip_ratio/low_mean": 6.996158772381023e-05, "clip_ratio/low_min": 6.996158772381023e-05, "clip_ratio/region_mean": 0.0011986046331003309, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 1982.0, "completions/mean_length": 1612.5599365234375, "completions/mean_terminated_length": 1425.94287109375, "completions/min_length": 736.0, "completions/min_terminated_length": 736.0, "entropy": 0.24363452196121216, "epoch": 0.019021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.01747559756040573, "learning_rate": 3e-05, "loss": 0.07337144017219543, "num_tokens": 624120.0, "reward": -0.08738280832767487, "reward_std": 0.6191282272338867, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.7873828411102295, "rewards/length_penalty/std": 0.19713006913661957, "sampling/importance_sampling_ratio/max": 1.55155348777771, "sampling/importance_sampling_ratio/mean": 0.9916171431541443, "sampling/importance_sampling_ratio/min": 0.6198675632476807, "sampling/sampling_logp_difference/max": 0.4782494306564331, "sampling/sampling_logp_difference/mean": 0.015680355951189995, "step": 7, "step_time": 27.640362513950095 }, { "clip_ratio/high_max": 0.0010140398750081657, "clip_ratio/high_mean": 0.0010140398750081657, "clip_ratio/low_mean": 2.1101497986819594e-05, "clip_ratio/low_min": 2.1101497986819594e-05, "clip_ratio/region_mean": 0.0010351413628086447, "completions/clipped_ratio": 0.5199999809265137, "completions/max_length": 2048.0, "completions/max_terminated_length": 1942.0, "completions/mean_length": 1791.219970703125, "completions/mean_terminated_length": 1513.041748046875, "completions/min_length": 731.0, "completions/min_terminated_length": 731.0, "entropy": 0.20876091718673706, "epoch": 0.021739130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.016676658764481544, "learning_rate": 3.5e-05, "loss": 0.0728004053235054, "num_tokens": 715831.0, "reward": -0.414619117975235, "reward_std": 0.6448422074317932, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.8746191263198853, "rewards/length_penalty/std": 0.16793787479400635, "sampling/importance_sampling_ratio/max": 1.458160638809204, "sampling/importance_sampling_ratio/mean": 0.9928096532821655, "sampling/importance_sampling_ratio/min": 0.013455379754304886, "sampling/sampling_logp_difference/max": 4.308376312255859, "sampling/sampling_logp_difference/mean": 0.014078265987336636, "step": 8, "step_time": 27.806979547720402 }, { "clip_ratio/high_max": 0.0004220485279802233, "clip_ratio/high_mean": 0.0004220485279802233, "clip_ratio/low_mean": 0.00018257334959344008, "clip_ratio/low_min": 0.00018257334959344008, "clip_ratio/region_mean": 0.0006046218739356845, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 1999.0, "completions/mean_length": 1643.47998046875, "completions/mean_terminated_length": 1470.1142578125, "completions/min_length": 814.0, "completions/min_terminated_length": 814.0, "entropy": 0.23288021981716156, "epoch": 0.024456521739130436, "frac_reward_zero_std": 0.0, "grad_norm": 0.0172340776771307, "learning_rate": 4e-05, "loss": 0.06772496551275253, "num_tokens": 800475.0, "reward": -0.2824804484844208, "reward_std": 0.6149426698684692, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.8024804592132568, "rewards/length_penalty/std": 0.1825057566165924, "sampling/importance_sampling_ratio/max": 1.7604615688323975, "sampling/importance_sampling_ratio/mean": 0.991741955280304, "sampling/importance_sampling_ratio/min": 0.5955495238304138, "sampling/sampling_logp_difference/max": 0.5655760765075684, "sampling/sampling_logp_difference/mean": 0.01497576106339693, "step": 9, "step_time": 27.90247388370335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1862.0, "completions/mean_length": 1401.0799560546875, "completions/mean_terminated_length": 1239.3499755859375, "completions/min_length": 726.0, "completions/min_terminated_length": 726.0, "entropy": 0.230853134393692, "epoch": 0.02717391304347826, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 4.5e-05, "loss": 0.0, "num_tokens": 875349.0, "reward": 0.11587890237569809, "reward_std": 0.5739556550979614, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.6841210722923279, "rewards/length_penalty/std": 0.2081541270017624, "sampling/importance_sampling_ratio/max": 1.4669582843780518, "sampling/importance_sampling_ratio/mean": 0.9917815327644348, "sampling/importance_sampling_ratio/min": 0.5419274568557739, "sampling/sampling_logp_difference/max": 0.6126232147216797, "sampling/sampling_logp_difference/mean": 0.015108686871826649, "step": 10, "step_time": 26.760133635252714 }, { "clip_ratio/high_max": 0.0006634370831307024, "clip_ratio/high_mean": 0.0006634370831307024, "clip_ratio/low_mean": 0.00011633349422481842, "clip_ratio/low_min": 0.00011633349422481842, "clip_ratio/region_mean": 0.000779770576627925, "completions/clipped_ratio": 0.5600000023841858, "completions/max_length": 2048.0, "completions/max_terminated_length": 2026.0, "completions/mean_length": 1909.89990234375, "completions/mean_terminated_length": 1734.136474609375, "completions/min_length": 1076.0, "completions/min_terminated_length": 1076.0, "entropy": 0.2466139942407608, "epoch": 0.029891304347826088, "frac_reward_zero_std": 0.0, "grad_norm": 0.017926953732967377, "learning_rate": 5e-05, "loss": 0.06259462237358093, "num_tokens": 973264.0, "reward": -0.4725683629512787, "reward_std": 0.5826442837715149, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.932568371295929, "rewards/length_penalty/std": 0.10813212394714355, "sampling/importance_sampling_ratio/max": 1.6338801383972168, "sampling/importance_sampling_ratio/mean": 0.9913659691810608, "sampling/importance_sampling_ratio/min": 0.5057844519615173, "sampling/sampling_logp_difference/max": 0.6816446781158447, "sampling/sampling_logp_difference/mean": 0.015841186046600342, "step": 11, "step_time": 28.522681473055854 }, { "clip_ratio/high_max": 0.0005377098161261529, "clip_ratio/high_mean": 0.0005377098161261529, "clip_ratio/low_mean": 0.00011695685898303055, "clip_ratio/low_min": 0.00011695685898303055, "clip_ratio/region_mean": 0.0006546666845679283, "completions/clipped_ratio": 0.3799999952316284, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 1734.7799072265625, "completions/mean_terminated_length": 1542.806396484375, "completions/min_length": 896.0, "completions/min_terminated_length": 896.0, "entropy": 0.27761703431606294, "epoch": 0.03260869565217391, "frac_reward_zero_std": 0.0, "grad_norm": 0.019899185746908188, "learning_rate": 5e-05, "loss": 0.062266021966934204, "num_tokens": 1062643.0, "reward": -0.22706054151058197, "reward_std": 0.6252650022506714, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.8470605611801147, "rewards/length_penalty/std": 0.17872333526611328, "sampling/importance_sampling_ratio/max": 1.7503340244293213, "sampling/importance_sampling_ratio/mean": 0.9904559254646301, "sampling/importance_sampling_ratio/min": 0.6366132497787476, "sampling/sampling_logp_difference/max": 0.5598065853118896, "sampling/sampling_logp_difference/mean": 0.017765674740076065, "step": 12, "step_time": 28.302239689277485 }, { "clip_ratio/high_max": 0.0004424852493684739, "clip_ratio/high_mean": 0.0004424852493684739, "clip_ratio/low_mean": 0.0001642594979784917, "clip_ratio/low_min": 0.0001642594979784917, "clip_ratio/region_mean": 0.0006067447306122631, "completions/clipped_ratio": 0.3199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 2028.0, "completions/mean_length": 1658.2799072265625, "completions/mean_terminated_length": 1474.88232421875, "completions/min_length": 764.0, "completions/min_terminated_length": 764.0, "entropy": 0.2525905281305313, "epoch": 0.035326086956521736, "frac_reward_zero_std": 0.0, "grad_norm": 0.018199454993009567, "learning_rate": 5e-05, "loss": 0.06363427639007568, "num_tokens": 1148577.0, "reward": -0.2297070324420929, "reward_std": 0.6296599507331848, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.8097070455551147, "rewards/length_penalty/std": 0.1831948459148407, "sampling/importance_sampling_ratio/max": 1.4388928413391113, "sampling/importance_sampling_ratio/mean": 0.991247296333313, "sampling/importance_sampling_ratio/min": 0.5558398962020874, "sampling/sampling_logp_difference/max": 0.5872750282287598, "sampling/sampling_logp_difference/mean": 0.016452128067612648, "step": 13, "step_time": 27.445479172980413 }, { "clip_ratio/high_max": 0.0007493688317481428, "clip_ratio/high_mean": 0.0007493688317481428, "clip_ratio/low_mean": 0.00011727154706022702, "clip_ratio/low_min": 0.00011727154706022702, "clip_ratio/region_mean": 0.0008666403999086469, "completions/clipped_ratio": 0.3799999952316284, "completions/max_length": 2048.0, "completions/max_terminated_length": 2038.0, "completions/mean_length": 1725.5999755859375, "completions/mean_terminated_length": 1528.0, "completions/min_length": 982.0, "completions/min_terminated_length": 982.0, "entropy": 0.24861081838607788, "epoch": 0.03804347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.01876567117869854, "learning_rate": 5e-05, "loss": 0.058530643582344055, "num_tokens": 1237417.0, "reward": -0.2025781273841858, "reward_std": 0.6181368827819824, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.842578113079071, "rewards/length_penalty/std": 0.17701490223407745, "sampling/importance_sampling_ratio/max": 2.049344539642334, "sampling/importance_sampling_ratio/mean": 0.9912201166152954, "sampling/importance_sampling_ratio/min": 0.6518958210945129, "sampling/sampling_logp_difference/max": 0.717519998550415, "sampling/sampling_logp_difference/mean": 0.015870127826929092, "step": 14, "step_time": 28.25221319217235 }, { "clip_ratio/high_max": 0.0005515470344107598, "clip_ratio/high_mean": 0.0005515470344107598, "clip_ratio/low_mean": 0.00011191076919203624, "clip_ratio/low_min": 0.00011191076919203624, "clip_ratio/region_mean": 0.0006634578050579876, "completions/clipped_ratio": 0.3799999952316284, "completions/max_length": 2048.0, "completions/max_terminated_length": 2005.0, "completions/mean_length": 1613.6199951171875, "completions/mean_terminated_length": 1347.3870849609375, "completions/min_length": 758.0, "completions/min_terminated_length": 758.0, "entropy": 0.20123647153377533, "epoch": 0.04076086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.01678749918937683, "learning_rate": 5e-05, "loss": 0.09095814824104309, "num_tokens": 1320678.0, "reward": -0.347900390625, "reward_std": 0.6158223152160645, "rewards/correctness/mean": 0.4399999976158142, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.7879003882408142, "rewards/length_penalty/std": 0.20662866532802582, "sampling/importance_sampling_ratio/max": 1.8356566429138184, "sampling/importance_sampling_ratio/mean": 0.9930147528648376, "sampling/importance_sampling_ratio/min": 0.6432644724845886, "sampling/sampling_logp_difference/max": 0.6074022054672241, "sampling/sampling_logp_difference/mean": 0.013288532383739948, "step": 15, "step_time": 27.802796632051468 }, { "clip_ratio/high_max": 0.00040561936039011924, "clip_ratio/high_mean": 0.00040561936039011924, "clip_ratio/low_mean": 0.00020730637043016032, "clip_ratio/low_min": 0.00020730637043016032, "clip_ratio/region_mean": 0.0006129257148131728, "completions/clipped_ratio": 0.3799999952316284, "completions/max_length": 2048.0, "completions/max_terminated_length": 2013.0, "completions/mean_length": 1647.199951171875, "completions/mean_terminated_length": 1401.54833984375, "completions/min_length": 819.0, "completions/min_terminated_length": 819.0, "entropy": 0.25868394374847414, "epoch": 0.043478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.01826147735118866, "learning_rate": 5e-05, "loss": 0.05328507721424103, "num_tokens": 1407138.0, "reward": -0.16429686546325684, "reward_std": 0.6472233533859253, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.8042968511581421, "rewards/length_penalty/std": 0.20003539323806763, "sampling/importance_sampling_ratio/max": 1.6819345951080322, "sampling/importance_sampling_ratio/mean": 0.9908729195594788, "sampling/importance_sampling_ratio/min": 0.20062674582004547, "sampling/sampling_logp_difference/max": 1.6063090562820435, "sampling/sampling_logp_difference/mean": 0.01680823415517807, "step": 16, "step_time": 27.277609845157713 }, { "clip_ratio/high_max": 0.0011126518482342362, "clip_ratio/high_mean": 0.0011126518482342362, "clip_ratio/low_mean": 4.591099204844795e-05, "clip_ratio/low_min": 4.591099204844795e-05, "clip_ratio/region_mean": 0.001158562849741429, "completions/clipped_ratio": 0.47999998927116394, "completions/max_length": 2048.0, "completions/max_terminated_length": 2015.0, "completions/mean_length": 1735.39990234375, "completions/mean_terminated_length": 1446.84619140625, "completions/min_length": 827.0, "completions/min_terminated_length": 827.0, "entropy": 0.2295297622680664, "epoch": 0.04619565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.018071748316287994, "learning_rate": 5e-05, "loss": 0.05114548280835152, "num_tokens": 1496558.0, "reward": -0.3273632824420929, "reward_std": 0.6652565002441406, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.847363293170929, "rewards/length_penalty/std": 0.19583332538604736, "sampling/importance_sampling_ratio/max": 1.784644365310669, "sampling/importance_sampling_ratio/mean": 0.9918433427810669, "sampling/importance_sampling_ratio/min": 0.5829048752784729, "sampling/sampling_logp_difference/max": 0.5792191028594971, "sampling/sampling_logp_difference/mean": 0.015029823407530785, "step": 17, "step_time": 27.705750009045005 }, { "clip_ratio/high_max": 0.0005537043092772364, "clip_ratio/high_mean": 0.0005537043092772364, "clip_ratio/low_mean": 0.00015489171346416697, "clip_ratio/low_min": 0.00015489171346416697, "clip_ratio/region_mean": 0.0007085960067342966, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 1942.0, "completions/mean_length": 1709.02001953125, "completions/mean_terminated_length": 1463.5517578125, "completions/min_length": 1014.0, "completions/min_terminated_length": 1014.0, "entropy": 0.2086818963289261, "epoch": 0.04891304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.018707159906625748, "learning_rate": 5e-05, "loss": 0.06255688518285751, "num_tokens": 1584509.0, "reward": -0.2544824182987213, "reward_std": 0.6470314264297485, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.8344824314117432, "rewards/length_penalty/std": 0.17634017765522003, "sampling/importance_sampling_ratio/max": 1.4237250089645386, "sampling/importance_sampling_ratio/mean": 0.9928529858589172, "sampling/importance_sampling_ratio/min": 0.6513714790344238, "sampling/sampling_logp_difference/max": 0.42867517471313477, "sampling/sampling_logp_difference/mean": 0.01369562093168497, "step": 18, "step_time": 28.102999786147848 }, { "clip_ratio/high_max": 0.00036137893912382426, "clip_ratio/high_mean": 0.00036137893912382426, "clip_ratio/low_mean": 0.00036555549886543306, "clip_ratio/low_min": 0.00036555549886543306, "clip_ratio/region_mean": 0.0007269344234373421, "completions/clipped_ratio": 0.5399999618530273, "completions/max_length": 2048.0, "completions/max_terminated_length": 1899.0, "completions/mean_length": 1781.52001953125, "completions/mean_terminated_length": 1468.6956787109375, "completions/min_length": 700.0, "completions/min_terminated_length": 700.0, "entropy": 0.28644888401031493, "epoch": 0.051630434782608696, "frac_reward_zero_std": 0.0, "grad_norm": 0.018079008907079697, "learning_rate": 5e-05, "loss": 0.06017068773508072, "num_tokens": 1677015.0, "reward": -0.4098828136920929, "reward_std": 0.6555013060569763, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.8698828220367432, "rewards/length_penalty/std": 0.18115760385990143, "sampling/importance_sampling_ratio/max": 1.7604482173919678, "sampling/importance_sampling_ratio/mean": 0.9899896383285522, "sampling/importance_sampling_ratio/min": 0.5442848205566406, "sampling/sampling_logp_difference/max": 0.6082825660705566, "sampling/sampling_logp_difference/mean": 0.01858239807188511, "step": 19, "step_time": 28.07069696020335 }, { "clip_ratio/high_max": 0.0012503917794674634, "clip_ratio/high_mean": 0.0012503917794674634, "clip_ratio/low_mean": 2.0936920918757096e-05, "clip_ratio/low_min": 2.0936920918757096e-05, "clip_ratio/region_mean": 0.001271328702569008, "completions/clipped_ratio": 0.5, "completions/max_length": 2048.0, "completions/max_terminated_length": 2009.0, "completions/mean_length": 1806.3800048828125, "completions/mean_terminated_length": 1564.760009765625, "completions/min_length": 744.0, "completions/min_terminated_length": 744.0, "entropy": 0.2616306632757187, "epoch": 0.05434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.017912009730935097, "learning_rate": 5e-05, "loss": 0.08223964273929596, "num_tokens": 1769924.0, "reward": -0.3820214867591858, "reward_std": 0.6335089206695557, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.8820214867591858, "rewards/length_penalty/std": 0.1758052408695221, "sampling/importance_sampling_ratio/max": 1.4523085355758667, "sampling/importance_sampling_ratio/mean": 0.9911108613014221, "sampling/importance_sampling_ratio/min": 0.6380840539932251, "sampling/sampling_logp_difference/max": 0.44928526878356934, "sampling/sampling_logp_difference/mean": 0.016677044332027435, "step": 20, "step_time": 28.512537690345198 }, { "clip_ratio/high_max": 0.0008731137961149216, "clip_ratio/high_mean": 0.0008731137961149216, "clip_ratio/low_mean": 4.748431238112971e-05, "clip_ratio/low_min": 4.748431238112971e-05, "clip_ratio/region_mean": 0.0009205981157720089, "completions/clipped_ratio": 0.35999998450279236, "completions/max_length": 2048.0, "completions/max_terminated_length": 2010.0, "completions/mean_length": 1731.2799072265625, "completions/mean_terminated_length": 1553.125, "completions/min_length": 1052.0, "completions/min_terminated_length": 1052.0, "entropy": 0.21868454813957214, "epoch": 0.057065217391304345, "frac_reward_zero_std": 0.0, "grad_norm": 0.01900572143495083, "learning_rate": 5e-05, "loss": 0.07679863274097443, "num_tokens": 1859898.0, "reward": -0.18535156548023224, "reward_std": 0.5985742211341858, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.47851815819740295, "rewards/length_penalty/mean": -0.8453515768051147, "rewards/length_penalty/std": 0.14833879470825195, "sampling/importance_sampling_ratio/max": 1.5182361602783203, "sampling/importance_sampling_ratio/mean": 0.9923606514930725, "sampling/importance_sampling_ratio/min": 0.639233410358429, "sampling/sampling_logp_difference/max": 0.44748568534851074, "sampling/sampling_logp_difference/mean": 0.014437779784202576, "step": 21, "step_time": 27.77762763411738 }, { "clip_ratio/high_max": 0.0011475141742266714, "clip_ratio/high_mean": 0.0011475141742266714, "clip_ratio/low_mean": 0.0001407445262884721, "clip_ratio/low_min": 0.0001407445262884721, "clip_ratio/region_mean": 0.0012882586801424623, "completions/clipped_ratio": 0.35999998450279236, "completions/max_length": 2048.0, "completions/max_terminated_length": 2046.0, "completions/mean_length": 1547.2999267578125, "completions/mean_terminated_length": 1265.65625, "completions/min_length": 698.0, "completions/min_terminated_length": 698.0, "entropy": 0.26379768550395966, "epoch": 0.059782608695652176, "frac_reward_zero_std": 0.0, "grad_norm": 0.0202660970389843, "learning_rate": 5e-05, "loss": 0.06397923082113266, "num_tokens": 1942333.0, "reward": -0.07551757991313934, "reward_std": 0.6593576669692993, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.4712120592594147, "rewards/length_penalty/mean": -0.7555176019668579, "rewards/length_penalty/std": 0.23032516241073608, "sampling/importance_sampling_ratio/max": 2.0485424995422363, "sampling/importance_sampling_ratio/mean": 0.9907693862915039, "sampling/importance_sampling_ratio/min": 0.6443278789520264, "sampling/sampling_logp_difference/max": 0.7171285152435303, "sampling/sampling_logp_difference/mean": 0.016709456220269203, "step": 22, "step_time": 28.331214781384915 }, { "clip_ratio/high_max": 0.0010714440606534481, "clip_ratio/high_mean": 0.0010714440606534481, "clip_ratio/low_mean": 0.00012812317872885616, "clip_ratio/low_min": 0.00012812317872885616, "clip_ratio/region_mean": 0.0011995672481134533, "completions/clipped_ratio": 0.47999998927116394, "completions/max_length": 2048.0, "completions/max_terminated_length": 1883.0, "completions/mean_length": 1720.5599365234375, "completions/mean_terminated_length": 1418.3077392578125, "completions/min_length": 1002.0, "completions/min_terminated_length": 1002.0, "entropy": 0.28058854341506956, "epoch": 0.0625, "frac_reward_zero_std": 0.0, "grad_norm": 0.01973128691315651, "learning_rate": 5e-05, "loss": 0.08977419883012772, "num_tokens": 2034481.0, "reward": -0.38011717796325684, "reward_std": 0.6433330178260803, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.8401172161102295, "rewards/length_penalty/std": 0.18192999064922333, "sampling/importance_sampling_ratio/max": 1.4594841003417969, "sampling/importance_sampling_ratio/mean": 0.9899795651435852, "sampling/importance_sampling_ratio/min": 0.5739880204200745, "sampling/sampling_logp_difference/max": 0.5551468133926392, "sampling/sampling_logp_difference/mean": 0.017823005095124245, "step": 23, "step_time": 28.871535916812718 }, { "clip_ratio/high_max": 0.0003421928733587265, "clip_ratio/high_mean": 0.0003421928733587265, "clip_ratio/low_mean": 0.00016381262175855227, "clip_ratio/low_min": 0.00016381262175855227, "clip_ratio/region_mean": 0.0005060054871137254, "completions/clipped_ratio": 0.5799999833106995, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1718.5399169921875, "completions/mean_terminated_length": 1263.5714111328125, "completions/min_length": 826.0, "completions/min_terminated_length": 826.0, "entropy": 0.25833035111427305, "epoch": 0.06521739130434782, "frac_reward_zero_std": 0.0, "grad_norm": 0.018430087715387344, "learning_rate": 5e-05, "loss": 0.07976283133029938, "num_tokens": 2122698.0, "reward": -0.43913084268569946, "reward_std": 0.7019431591033936, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.8391308784484863, "rewards/length_penalty/std": 0.23197655379772186, "sampling/importance_sampling_ratio/max": 1.4506571292877197, "sampling/importance_sampling_ratio/mean": 0.9910207390785217, "sampling/importance_sampling_ratio/min": 0.6690610647201538, "sampling/sampling_logp_difference/max": 0.4018799066543579, "sampling/sampling_logp_difference/mean": 0.016251178458333015, "step": 24, "step_time": 28.187903855461627 }, { "clip_ratio/high_max": 0.00030475384555757047, "clip_ratio/high_mean": 0.00030475384555757047, "clip_ratio/low_mean": 0.00024887076287996026, "clip_ratio/low_min": 0.00024887076287996026, "clip_ratio/region_mean": 0.0005536246113479138, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 1982.0, "completions/mean_length": 1367.0, "completions/mean_terminated_length": 1174.923095703125, "completions/min_length": 589.0, "completions/min_terminated_length": 589.0, "entropy": 0.2877860367298126, "epoch": 0.06793478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.02185056358575821, "learning_rate": 5e-05, "loss": 0.04259161278605461, "num_tokens": 2193448.0, "reward": 0.1125195324420929, "reward_std": 0.6123260259628296, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.66748046875, "rewards/length_penalty/std": 0.22482573986053467, "sampling/importance_sampling_ratio/max": 1.458466649055481, "sampling/importance_sampling_ratio/mean": 0.9898408055305481, "sampling/importance_sampling_ratio/min": 0.5802799463272095, "sampling/sampling_logp_difference/max": 0.544244647026062, "sampling/sampling_logp_difference/mean": 0.01890159212052822, "step": 25, "step_time": 26.1629780868534 }, { "clip_ratio/high_max": 0.0004262695380020887, "clip_ratio/high_mean": 0.0004262695380020887, "clip_ratio/low_mean": 0.00010179920209338889, "clip_ratio/low_min": 0.00010179920209338889, "clip_ratio/region_mean": 0.0005280687459162436, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 2012.0, "completions/mean_length": 1366.6199951171875, "completions/mean_terminated_length": 1290.9111328125, "completions/min_length": 718.0, "completions/min_terminated_length": 718.0, "entropy": 0.2429080605506897, "epoch": 0.07065217391304347, "frac_reward_zero_std": 0.0, "grad_norm": 0.019393621012568474, "learning_rate": 5e-05, "loss": 0.0699596256017685, "num_tokens": 2264249.0, "reward": -0.14729492366313934, "reward_std": 0.5188794136047363, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.6672949194908142, "rewards/length_penalty/std": 0.17943644523620605, "sampling/importance_sampling_ratio/max": 1.6634753942489624, "sampling/importance_sampling_ratio/mean": 0.9914184212684631, "sampling/importance_sampling_ratio/min": 0.5688079595565796, "sampling/sampling_logp_difference/max": 0.5642123222351074, "sampling/sampling_logp_difference/mean": 0.016148291528224945, "step": 26, "step_time": 26.751327357254922 }, { "clip_ratio/high_max": 0.00023492113687098026, "clip_ratio/high_mean": 0.00023492113687098026, "clip_ratio/low_mean": 0.00018710011499933898, "clip_ratio/low_min": 0.00018710011499933898, "clip_ratio/region_mean": 0.0004220212562358938, "completions/clipped_ratio": 0.3999999761581421, "completions/max_length": 2048.0, "completions/max_terminated_length": 1956.0, "completions/mean_length": 1536.2999267578125, "completions/mean_terminated_length": 1195.166748046875, "completions/min_length": 765.0, "completions/min_terminated_length": 765.0, "entropy": 0.20874288380146028, "epoch": 0.07336956521739131, "frac_reward_zero_std": 0.0, "grad_norm": 0.018195312470197678, "learning_rate": 5e-05, "loss": 0.04994223266839981, "num_tokens": 2343314.0, "reward": -0.150146484375, "reward_std": 0.7131717205047607, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.7501465082168579, "rewards/length_penalty/std": 0.24444596469402313, "sampling/importance_sampling_ratio/max": 1.5101569890975952, "sampling/importance_sampling_ratio/mean": 0.9925653338432312, "sampling/importance_sampling_ratio/min": 0.5024821758270264, "sampling/sampling_logp_difference/max": 0.6881951093673706, "sampling/sampling_logp_difference/mean": 0.013967176899313927, "step": 27, "step_time": 26.71739569515921 }, { "clip_ratio/high_max": 0.0006563825038028881, "clip_ratio/high_mean": 0.0006563825038028881, "clip_ratio/low_mean": 0.0001403340545948595, "clip_ratio/low_min": 0.0001403340545948595, "clip_ratio/region_mean": 0.0007967165671288967, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 2042.0, "completions/mean_length": 1280.47998046875, "completions/mean_terminated_length": 1231.4893798828125, "completions/min_length": 661.0, "completions/min_terminated_length": 661.0, "entropy": 0.19260205030441285, "epoch": 0.07608695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.017330382019281387, "learning_rate": 5e-05, "loss": 0.04802786931395531, "num_tokens": 2410698.0, "reward": 0.134765625, "reward_std": 0.46675702929496765, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.6252343654632568, "rewards/length_penalty/std": 0.17892423272132874, "sampling/importance_sampling_ratio/max": 1.742319941520691, "sampling/importance_sampling_ratio/mean": 0.9931791424751282, "sampling/importance_sampling_ratio/min": 0.5006048679351807, "sampling/sampling_logp_difference/max": 0.6919381618499756, "sampling/sampling_logp_difference/mean": 0.013207054696977139, "step": 28, "step_time": 26.334424772299826 }, { "clip_ratio/high_max": 0.0006336055404972285, "clip_ratio/high_mean": 0.0006336055404972285, "clip_ratio/low_mean": 0.00016180542879737914, "clip_ratio/low_min": 0.00016180542879737914, "clip_ratio/region_mean": 0.0007954109751153737, "completions/clipped_ratio": 0.4599999785423279, "completions/max_length": 2048.0, "completions/max_terminated_length": 2042.0, "completions/mean_length": 1685.4599609375, "completions/mean_terminated_length": 1376.629638671875, "completions/min_length": 569.0, "completions/min_terminated_length": 569.0, "entropy": 0.2433333694934845, "epoch": 0.07880434782608696, "frac_reward_zero_std": 0.0, "grad_norm": 0.021047445014119148, "learning_rate": 5e-05, "loss": 0.04609709233045578, "num_tokens": 2498411.0, "reward": -0.2429785132408142, "reward_std": 0.6770392656326294, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.8229784965515137, "rewards/length_penalty/std": 0.24099376797676086, "sampling/importance_sampling_ratio/max": 1.7422999143600464, "sampling/importance_sampling_ratio/mean": 0.9914277791976929, "sampling/importance_sampling_ratio/min": 0.45384329557418823, "sampling/sampling_logp_difference/max": 0.7900032997131348, "sampling/sampling_logp_difference/mean": 0.01619006134569645, "step": 29, "step_time": 27.966543859802186 }, { "clip_ratio/high_max": 0.00032839191262610255, "clip_ratio/high_mean": 0.00032839191262610255, "clip_ratio/low_mean": 0.00017811718716984615, "clip_ratio/low_min": 0.00017811718716984615, "clip_ratio/region_mean": 0.0005065091070719063, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 2026.0, "completions/mean_length": 1343.3800048828125, "completions/mean_terminated_length": 1247.2955322265625, "completions/min_length": 746.0, "completions/min_terminated_length": 746.0, "entropy": 0.17178598344326018, "epoch": 0.08152173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.01862315647304058, "learning_rate": 5e-05, "loss": 0.05354345962405205, "num_tokens": 2567370.0, "reward": 0.22405272722244263, "reward_std": 0.48565441370010376, "rewards/correctness/mean": 0.8799999952316284, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.6559472680091858, "rewards/length_penalty/std": 0.20939743518829346, "sampling/importance_sampling_ratio/max": 1.7974625825881958, "sampling/importance_sampling_ratio/mean": 0.9936282634735107, "sampling/importance_sampling_ratio/min": 0.5518352389335632, "sampling/sampling_logp_difference/max": 0.594505786895752, "sampling/sampling_logp_difference/mean": 0.012440630234777927, "step": 30, "step_time": 25.884617429226637 }, { "clip_ratio/high_max": 0.0003395844076294452, "clip_ratio/high_mean": 0.0003395844076294452, "clip_ratio/low_mean": 0.0001251072193554137, "clip_ratio/low_min": 0.0001251072193554137, "clip_ratio/region_mean": 0.00046469162334688007, "completions/clipped_ratio": 0.35999998450279236, "completions/max_length": 2048.0, "completions/max_terminated_length": 2000.0, "completions/mean_length": 1658.1199951171875, "completions/mean_terminated_length": 1438.8125, "completions/min_length": 642.0, "completions/min_terminated_length": 642.0, "entropy": 0.19451322555541992, "epoch": 0.08423913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.017832906916737556, "learning_rate": 5e-05, "loss": 0.05964586138725281, "num_tokens": 2653406.0, "reward": -0.5296288728713989, "reward_std": 0.48112568259239197, "rewards/correctness/mean": 0.2800000011920929, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.8096289038658142, "rewards/length_penalty/std": 0.2021900862455368, "sampling/importance_sampling_ratio/max": 1.5269027948379517, "sampling/importance_sampling_ratio/mean": 0.9930987358093262, "sampling/importance_sampling_ratio/min": 0.5175579786300659, "sampling/sampling_logp_difference/max": 0.6586337089538574, "sampling/sampling_logp_difference/mean": 0.013245755806565285, "step": 31, "step_time": 28.08954193815589 }, { "clip_ratio/high_max": 0.0006213331886101514, "clip_ratio/high_mean": 0.0006213331886101514, "clip_ratio/low_mean": 0.0001274906549951993, "clip_ratio/low_min": 0.0001274906549951993, "clip_ratio/region_mean": 0.00074882383341901, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 2041.0, "completions/mean_length": 1299.9000244140625, "completions/mean_terminated_length": 1178.1163330078125, "completions/min_length": 575.0, "completions/min_terminated_length": 575.0, "entropy": 0.2351977288722992, "epoch": 0.08695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.018212830647826195, "learning_rate": 5e-05, "loss": 0.0910167321562767, "num_tokens": 2720331.0, "reward": 0.20528319478034973, "reward_std": 0.5360139012336731, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.634716808795929, "rewards/length_penalty/std": 0.21133719384670258, "sampling/importance_sampling_ratio/max": 1.80198335647583, "sampling/importance_sampling_ratio/mean": 0.9916144013404846, "sampling/importance_sampling_ratio/min": 0.4718713164329529, "sampling/sampling_logp_difference/max": 0.7510490417480469, "sampling/sampling_logp_difference/mean": 0.016335755586624146, "step": 32, "step_time": 25.710428322665393 }, { "clip_ratio/high_max": 0.00021464402670972048, "clip_ratio/high_mean": 0.00021464402670972048, "clip_ratio/low_mean": 0.00015989402163540944, "clip_ratio/low_min": 0.00015989402163540944, "clip_ratio/region_mean": 0.00037453805416589605, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 2020.0, "completions/mean_length": 1479.4599609375, "completions/mean_terminated_length": 1235.800048828125, "completions/min_length": 846.0, "completions/min_terminated_length": 846.0, "entropy": 0.20212584137916564, "epoch": 0.08967391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.01504465937614441, "learning_rate": 5e-05, "loss": 0.019914399832487106, "num_tokens": 2797944.0, "reward": -0.18239256739616394, "reward_std": 0.6438302397727966, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.7223925590515137, "rewards/length_penalty/std": 0.23767130076885223, "sampling/importance_sampling_ratio/max": 1.7032686471939087, "sampling/importance_sampling_ratio/mean": 0.9927703738212585, "sampling/importance_sampling_ratio/min": 0.48547396063804626, "sampling/sampling_logp_difference/max": 0.7226296663284302, "sampling/sampling_logp_difference/mean": 0.013833415694534779, "step": 33, "step_time": 27.364153544185683 }, { "clip_ratio/high_max": 0.0007488755392841995, "clip_ratio/high_mean": 0.0007488755392841995, "clip_ratio/low_mean": 0.00013347235508263112, "clip_ratio/low_min": 0.00013347235508263112, "clip_ratio/region_mean": 0.0008823479060083627, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 1869.0, "completions/mean_length": 1504.5999755859375, "completions/mean_terminated_length": 1293.27783203125, "completions/min_length": 760.0, "completions/min_terminated_length": 760.0, "entropy": 0.23613905310630798, "epoch": 0.09239130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.019435470923781395, "learning_rate": 5e-05, "loss": 0.06828445196151733, "num_tokens": 2877064.0, "reward": -0.03466796875, "reward_std": 0.6289466619491577, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.734667956829071, "rewards/length_penalty/std": 0.19785979390144348, "sampling/importance_sampling_ratio/max": 2.3225607872009277, "sampling/importance_sampling_ratio/mean": 0.9916264414787292, "sampling/importance_sampling_ratio/min": 0.4259100556373596, "sampling/sampling_logp_difference/max": 0.8535270690917969, "sampling/sampling_logp_difference/mean": 0.01610179990530014, "step": 34, "step_time": 27.096626644255593 }, { "clip_ratio/high_max": 0.00018833643407560884, "clip_ratio/high_mean": 0.00018833643407560884, "clip_ratio/low_mean": 0.00023152987123467028, "clip_ratio/low_min": 0.00023152987123467028, "clip_ratio/region_mean": 0.0004198663111310452, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1908.0, "completions/mean_length": 1410.659912109375, "completions/mean_terminated_length": 1251.3250732421875, "completions/min_length": 805.0, "completions/min_terminated_length": 805.0, "entropy": 0.2762973129749298, "epoch": 0.09510869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.02337278425693512, "learning_rate": 5e-05, "loss": 0.053426593542099, "num_tokens": 2950327.0, "reward": 0.09120117127895355, "reward_std": 0.5909886956214905, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.6887988448143005, "rewards/length_penalty/std": 0.20144735276699066, "sampling/importance_sampling_ratio/max": 2.8600780963897705, "sampling/importance_sampling_ratio/mean": 0.9903241395950317, "sampling/importance_sampling_ratio/min": 0.24700671434402466, "sampling/sampling_logp_difference/max": 1.3983397483825684, "sampling/sampling_logp_difference/mean": 0.01899528317153454, "step": 35, "step_time": 26.863242340972647 }, { "clip_ratio/high_max": 0.0002487683479557745, "clip_ratio/high_mean": 0.0002487683479557745, "clip_ratio/low_mean": 7.831925177015364e-05, "clip_ratio/low_min": 7.831925177015364e-05, "clip_ratio/region_mean": 0.0003270876011811197, "completions/clipped_ratio": 0.17999999225139618, "completions/max_length": 2048.0, "completions/max_terminated_length": 1986.0, "completions/mean_length": 1303.679931640625, "completions/mean_terminated_length": 1140.2926025390625, "completions/min_length": 643.0, "completions/min_terminated_length": 643.0, "entropy": 0.2200786739587784, "epoch": 0.09782608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.02034231461584568, "learning_rate": 5e-05, "loss": 0.03495493531227112, "num_tokens": 3017981.0, "reward": 0.16343750059604645, "reward_std": 0.5867263674736023, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.6365625262260437, "rewards/length_penalty/std": 0.2089543491601944, "sampling/importance_sampling_ratio/max": 2.8404994010925293, "sampling/importance_sampling_ratio/mean": 0.9921259880065918, "sampling/importance_sampling_ratio/min": 0.3215929865837097, "sampling/sampling_logp_difference/max": 1.1344685554504395, "sampling/sampling_logp_difference/mean": 0.016007130965590477, "step": 36, "step_time": 26.02595334779471 }, { "clip_ratio/high_max": 0.000559749428066425, "clip_ratio/high_mean": 0.000559749428066425, "clip_ratio/low_mean": 0.00018818537355400622, "clip_ratio/low_min": 0.00018818537355400622, "clip_ratio/region_mean": 0.0007479347928892821, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 1980.0, "completions/mean_length": 1476.239990234375, "completions/mean_terminated_length": 1253.888916015625, "completions/min_length": 809.0, "completions/min_terminated_length": 809.0, "entropy": 0.2496323198080063, "epoch": 0.10054347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.02047264389693737, "learning_rate": 5e-05, "loss": 0.056008629500865936, "num_tokens": 3095313.0, "reward": -0.1808203011751175, "reward_std": 0.6257206797599792, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.7208203077316284, "rewards/length_penalty/std": 0.21440976858139038, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.990962028503418, "sampling/importance_sampling_ratio/min": 0.3304795026779175, "sampling/sampling_logp_difference/max": 1.107210636138916, "sampling/sampling_logp_difference/mean": 0.017618639394640923, "step": 37, "step_time": 26.947069013724104 }, { "clip_ratio/high_max": 0.000511182751506567, "clip_ratio/high_mean": 0.000511182751506567, "clip_ratio/low_mean": 6.747063307557254e-05, "clip_ratio/low_min": 6.747063307557254e-05, "clip_ratio/region_mean": 0.0005786533816717565, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1980.0, "completions/mean_length": 1477.0799560546875, "completions/mean_terminated_length": 1334.3499755859375, "completions/min_length": 735.0, "completions/min_terminated_length": 735.0, "entropy": 0.17609985172748566, "epoch": 0.10326086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.019006526097655296, "learning_rate": 5e-05, "loss": 0.05077965557575226, "num_tokens": 3172557.0, "reward": 0.09876953065395355, "reward_std": 0.5452290177345276, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.7212304472923279, "rewards/length_penalty/std": 0.22814445197582245, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9937909245491028, "sampling/importance_sampling_ratio/min": 0.3079715967178345, "sampling/sampling_logp_difference/max": 1.1777477264404297, "sampling/sampling_logp_difference/mean": 0.013070470653474331, "step": 38, "step_time": 25.91789171216078 }, { "clip_ratio/high_max": 0.001432780921459198, "clip_ratio/high_mean": 0.001432780921459198, "clip_ratio/low_mean": 4.17014176491648e-05, "clip_ratio/low_min": 4.17014176491648e-05, "clip_ratio/region_mean": 0.0014744823332875968, "completions/clipped_ratio": 0.23999999463558197, "completions/max_length": 2048.0, "completions/max_terminated_length": 1794.0, "completions/mean_length": 1245.280029296875, "completions/mean_terminated_length": 991.7894897460938, "completions/min_length": 576.0, "completions/min_terminated_length": 576.0, "entropy": 0.20301035642623902, "epoch": 0.10597826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.021479694172739983, "learning_rate": 5e-05, "loss": 0.09315389394760132, "num_tokens": 3237311.0, "reward": -0.04804687574505806, "reward_std": 0.7439601421356201, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.6080468893051147, "rewards/length_penalty/std": 0.2620549201965332, "sampling/importance_sampling_ratio/max": 1.7770084142684937, "sampling/importance_sampling_ratio/mean": 0.9925133585929871, "sampling/importance_sampling_ratio/min": 0.24916520714759827, "sampling/sampling_logp_difference/max": 1.389639139175415, "sampling/sampling_logp_difference/mean": 0.014246346428990364, "step": 39, "step_time": 24.72481947694905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1863.0, "completions/max_terminated_length": 1863.0, "completions/mean_length": 996.7599487304688, "completions/mean_terminated_length": 996.7599487304688, "completions/min_length": 679.0, "completions/min_terminated_length": 679.0, "entropy": 0.19697461724281312, "epoch": 0.10869565217391304, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 3289869.0, "reward": 0.5133007764816284, "reward_std": 0.1261473447084427, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.4866992235183716, "rewards/length_penalty/std": 0.1261473447084427, "sampling/importance_sampling_ratio/max": 2.2436959743499756, "sampling/importance_sampling_ratio/mean": 0.992948055267334, "sampling/importance_sampling_ratio/min": 0.18371763825416565, "sampling/sampling_logp_difference/max": 1.6943552494049072, "sampling/sampling_logp_difference/mean": 0.015295255929231644, "step": 40, "step_time": 22.31505933124572 }, { "clip_ratio/high_max": 3.0564682674594223e-05, "clip_ratio/high_mean": 3.0564682674594223e-05, "clip_ratio/low_mean": 0.00022323383891489356, "clip_ratio/low_min": 0.00022323383891489356, "clip_ratio/region_mean": 0.0002537985215894878, "completions/clipped_ratio": 0.23999999463558197, "completions/max_length": 2048.0, "completions/max_terminated_length": 1801.0, "completions/mean_length": 1355.8199462890625, "completions/mean_terminated_length": 1137.23681640625, "completions/min_length": 602.0, "completions/min_terminated_length": 602.0, "entropy": 0.20732976198196412, "epoch": 0.11141304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.017952512949705124, "learning_rate": 5e-05, "loss": 0.0316755548119545, "num_tokens": 3361400.0, "reward": -0.22202147543430328, "reward_std": 0.6473181843757629, "rewards/correctness/mean": 0.4399999976158142, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.6620214581489563, "rewards/length_penalty/std": 0.22592417895793915, "sampling/importance_sampling_ratio/max": 2.6353089809417725, "sampling/importance_sampling_ratio/mean": 0.992411196231842, "sampling/importance_sampling_ratio/min": 0.35229283571243286, "sampling/sampling_logp_difference/max": 1.04329252243042, "sampling/sampling_logp_difference/mean": 0.01496540941298008, "step": 41, "step_time": 25.674194851191714 }, { "clip_ratio/high_max": 0.0010855704254936426, "clip_ratio/high_mean": 0.0010855704254936426, "clip_ratio/low_mean": 0.00010215246147708968, "clip_ratio/low_min": 0.00010215246147708968, "clip_ratio/region_mean": 0.0011877228913363068, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1978.0, "completions/mean_length": 1358.199951171875, "completions/mean_terminated_length": 1115.837890625, "completions/min_length": 584.0, "completions/min_terminated_length": 584.0, "entropy": 0.21488374769687651, "epoch": 0.11413043478260869, "frac_reward_zero_std": 0.0, "grad_norm": 0.019145222380757332, "learning_rate": 5e-05, "loss": 0.017716169357299805, "num_tokens": 3432300.0, "reward": 0.07681640237569809, "reward_std": 0.6722453236579895, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.6631835699081421, "rewards/length_penalty/std": 0.27725136280059814, "sampling/importance_sampling_ratio/max": 2.4651687145233154, "sampling/importance_sampling_ratio/mean": 0.9922519326210022, "sampling/importance_sampling_ratio/min": 0.2005905658006668, "sampling/sampling_logp_difference/max": 1.6064894199371338, "sampling/sampling_logp_difference/mean": 0.01567744091153145, "step": 42, "step_time": 25.395748630864546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1853.0, "completions/max_terminated_length": 1853.0, "completions/mean_length": 1045.47998046875, "completions/mean_terminated_length": 1045.47998046875, "completions/min_length": 568.0, "completions/min_terminated_length": 568.0, "entropy": 0.22891799807548524, "epoch": 0.11684782608695653, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 3486544.0, "reward": 0.4895116984844208, "reward_std": 0.16702203452587128, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.5104882717132568, "rewards/length_penalty/std": 0.16702201962471008, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9918332695960999, "sampling/importance_sampling_ratio/min": 0.23587487637996674, "sampling/sampling_logp_difference/max": 1.4444538354873657, "sampling/sampling_logp_difference/mean": 0.01812758482992649, "step": 43, "step_time": 22.243782210163772 }, { "clip_ratio/high_max": 0.0015327420551329851, "clip_ratio/high_mean": 0.0015327420551329851, "clip_ratio/low_mean": 1.4509576431009918e-05, "clip_ratio/low_min": 1.4509576431009918e-05, "clip_ratio/region_mean": 0.0015472516417503357, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 2030.0, "completions/mean_length": 1294.679931640625, "completions/mean_terminated_length": 1191.95458984375, "completions/min_length": 739.0, "completions/min_terminated_length": 739.0, "entropy": 0.21075761318206787, "epoch": 0.11956521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.020982731133699417, "learning_rate": 5e-05, "loss": 0.04945548623800278, "num_tokens": 3554278.0, "reward": 0.2678320109844208, "reward_std": 0.4623929262161255, "rewards/correctness/mean": 0.8999999761581421, "rewards/correctness/std": 0.30304574966430664, "rewards/length_penalty/mean": -0.6321679949760437, "rewards/length_penalty/std": 0.21657036244869232, "sampling/importance_sampling_ratio/max": 2.3799331188201904, "sampling/importance_sampling_ratio/mean": 0.9923880100250244, "sampling/importance_sampling_ratio/min": 0.23559124767780304, "sampling/sampling_logp_difference/max": 1.4456570148468018, "sampling/sampling_logp_difference/mean": 0.015859467908740044, "step": 44, "step_time": 25.036265914794058 }, { "clip_ratio/high_max": 0.0015714343637228013, "clip_ratio/high_mean": 0.0015714343637228013, "clip_ratio/low_mean": 1.6802486788947137e-05, "clip_ratio/low_min": 1.6802486788947137e-05, "clip_ratio/region_mean": 0.0015882368432357907, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 1926.0, "completions/mean_length": 1179.0999755859375, "completions/mean_terminated_length": 1013.5952758789062, "completions/min_length": 482.0, "completions/min_terminated_length": 482.0, "entropy": 0.22721909880638122, "epoch": 0.12228260869565218, "frac_reward_zero_std": 0.0, "grad_norm": 0.018693450838327408, "learning_rate": 5e-05, "loss": -0.005135001614689827, "num_tokens": 3616173.0, "reward": 0.14426757395267487, "reward_std": 0.6730633974075317, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.575732409954071, "rewards/length_penalty/std": 0.2690422534942627, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9917641878128052, "sampling/importance_sampling_ratio/min": 0.16028359532356262, "sampling/sampling_logp_difference/max": 1.830810546875, "sampling/sampling_logp_difference/mean": 0.01722223311662674, "step": 45, "step_time": 25.075448021758348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1900.0, "completions/mean_length": 1270.419921875, "completions/mean_terminated_length": 1076.0250244140625, "completions/min_length": 684.0, "completions/min_terminated_length": 684.0, "entropy": 0.23793226182460786, "epoch": 0.125, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 3683204.0, "reward": 0.17967772483825684, "reward_std": 0.6086320877075195, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.6203222870826721, "rewards/length_penalty/std": 0.22846709191799164, "sampling/importance_sampling_ratio/max": 2.4910104274749756, "sampling/importance_sampling_ratio/mean": 0.9915054440498352, "sampling/importance_sampling_ratio/min": 0.19439814984798431, "sampling/sampling_logp_difference/max": 1.6378469467163086, "sampling/sampling_logp_difference/mean": 0.01706322655081749, "step": 46, "step_time": 25.01791040482931 }, { "clip_ratio/high_max": 0.0005431722558569164, "clip_ratio/high_mean": 0.0005431722558569164, "clip_ratio/low_mean": 9.327361331088469e-05, "clip_ratio/low_min": 9.327361331088469e-05, "clip_ratio/region_mean": 0.0006364458822645247, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1863.0, "completions/mean_length": 1039.8399658203125, "completions/mean_terminated_length": 927.822265625, "completions/min_length": 615.0, "completions/min_terminated_length": 615.0, "entropy": 0.20186964869499208, "epoch": 0.12771739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.02016916126012802, "learning_rate": 5e-05, "loss": 0.08864342421293259, "num_tokens": 3737586.0, "reward": 0.37226560711860657, "reward_std": 0.5214666128158569, "rewards/correctness/mean": 0.8799999952316284, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.5077343583106995, "rewards/length_penalty/std": 0.21749594807624817, "sampling/importance_sampling_ratio/max": 2.4688563346862793, "sampling/importance_sampling_ratio/mean": 0.992901086807251, "sampling/importance_sampling_ratio/min": 0.12217313051223755, "sampling/sampling_logp_difference/max": 2.10231614112854, "sampling/sampling_logp_difference/mean": 0.015567263588309288, "step": 47, "step_time": 23.965738210827112 }, { "clip_ratio/high_max": 0.0010878464090637863, "clip_ratio/high_mean": 0.0010878464090637863, "clip_ratio/low_mean": 4.96484964969568e-05, "clip_ratio/low_min": 4.96484964969568e-05, "clip_ratio/region_mean": 0.0011374948895536362, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 2005.0, "completions/mean_length": 1220.97998046875, "completions/mean_terminated_length": 1063.452392578125, "completions/min_length": 516.0, "completions/min_terminated_length": 516.0, "entropy": 0.20233306884765626, "epoch": 0.13043478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.02060714364051819, "learning_rate": 5e-05, "loss": 0.07064387947320938, "num_tokens": 3801595.0, "reward": 0.06381835788488388, "reward_std": 0.6489872336387634, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.4785180985927582, "rewards/length_penalty/mean": -0.5961816310882568, "rewards/length_penalty/std": 0.24806849658489227, "sampling/importance_sampling_ratio/max": 2.319500207901001, "sampling/importance_sampling_ratio/mean": 0.9927141666412354, "sampling/importance_sampling_ratio/min": 0.24584947526454926, "sampling/sampling_logp_difference/max": 1.4030358791351318, "sampling/sampling_logp_difference/mean": 0.015085207298398018, "step": 48, "step_time": 25.31422982318327 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1419.0, "completions/mean_length": 1180.3399658203125, "completions/mean_terminated_length": 963.4249877929688, "completions/min_length": 558.0, "completions/min_terminated_length": 558.0, "entropy": 0.24282355904579161, "epoch": 0.1331521739130435, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 3863302.0, "reward": 0.2236621081829071, "reward_std": 0.6263456344604492, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.5763378739356995, "rewards/length_penalty/std": 0.23689721524715424, "sampling/importance_sampling_ratio/max": 2.755671501159668, "sampling/importance_sampling_ratio/mean": 0.9916248917579651, "sampling/importance_sampling_ratio/min": 0.168172225356102, "sampling/sampling_logp_difference/max": 1.7827666997909546, "sampling/sampling_logp_difference/mean": 0.018267333507537842, "step": 49, "step_time": 24.52096749516204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1238.0, "completions/mean_length": 1112.1400146484375, "completions/mean_terminated_length": 878.1749877929688, "completions/min_length": 435.0, "completions/min_terminated_length": 435.0, "entropy": 0.2699510782957077, "epoch": 0.1358695652173913, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 3921319.0, "reward": 0.056962888687849045, "reward_std": 0.6754798293113708, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.5430371165275574, "rewards/length_penalty/std": 0.24609321355819702, "sampling/importance_sampling_ratio/max": 2.7951529026031494, "sampling/importance_sampling_ratio/mean": 0.9901461005210876, "sampling/importance_sampling_ratio/min": 0.23735840618610382, "sampling/sampling_logp_difference/max": 1.4381840229034424, "sampling/sampling_logp_difference/mean": 0.019994623959064484, "step": 50, "step_time": 24.279819048009813 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1869.0, "completions/max_terminated_length": 1869.0, "completions/mean_length": 1052.679931640625, "completions/mean_terminated_length": 1052.679931640625, "completions/min_length": 606.0, "completions/min_terminated_length": 606.0, "entropy": 0.20157283544540405, "epoch": 0.13858695652173914, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 3976463.0, "reward": 0.4859960973262787, "reward_std": 0.16966456174850464, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.5140039324760437, "rewards/length_penalty/std": 0.16966456174850464, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9927499294281006, "sampling/importance_sampling_ratio/min": 0.12030817568302155, "sampling/sampling_logp_difference/max": 2.1176986694335938, "sampling/sampling_logp_difference/mean": 0.016359582543373108, "step": 51, "step_time": 22.70568755082786 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1939.0, "completions/max_terminated_length": 1939.0, "completions/mean_length": 989.97998046875, "completions/mean_terminated_length": 989.97998046875, "completions/min_length": 618.0, "completions/min_terminated_length": 618.0, "entropy": 0.18855113089084624, "epoch": 0.14130434782608695, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4028222.0, "reward": 0.31661131978034973, "reward_std": 0.39669328927993774, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.48338866233825684, "rewards/length_penalty/std": 0.15820050239562988, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9933652281761169, "sampling/importance_sampling_ratio/min": 0.1932464987039566, "sampling/sampling_logp_difference/max": 1.6437886953353882, "sampling/sampling_logp_difference/mean": 0.01552127581089735, "step": 52, "step_time": 22.332435484975576 }, { "clip_ratio/high_max": 0.0003080288151977584, "clip_ratio/high_mean": 0.0003080288151977584, "clip_ratio/low_mean": 7.110195583663881e-05, "clip_ratio/low_min": 7.110195583663881e-05, "clip_ratio/region_mean": 0.0003791307666688226, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 2034.0, "completions/mean_length": 1261.219970703125, "completions/mean_terminated_length": 1133.1395263671875, "completions/min_length": 567.0, "completions/min_terminated_length": 567.0, "entropy": 0.23178259730339051, "epoch": 0.14402173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.023617003113031387, "learning_rate": 5e-05, "loss": 0.10813619196414948, "num_tokens": 4094003.0, "reward": 0.024169921875, "reward_std": 0.6831474304199219, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.6158300638198853, "rewards/length_penalty/std": 0.23983381688594818, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9915435910224915, "sampling/importance_sampling_ratio/min": 0.14022201299667358, "sampling/sampling_logp_difference/max": 1.9645283222198486, "sampling/sampling_logp_difference/mean": 0.018086407333612442, "step": 53, "step_time": 24.88868365692906 }, { "clip_ratio/high_max": 0.0007304294151253999, "clip_ratio/high_mean": 0.0007304294151253999, "clip_ratio/low_mean": 0.00010322023881599307, "clip_ratio/low_min": 0.00010322023881599307, "clip_ratio/region_mean": 0.0008336496539413929, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 2000.0, "completions/mean_length": 1351.6400146484375, "completions/mean_terminated_length": 1155.2308349609375, "completions/min_length": 669.0, "completions/min_terminated_length": 669.0, "entropy": 0.2325251430273056, "epoch": 0.14673913043478262, "frac_reward_zero_std": 0.0, "grad_norm": 0.021172959357500076, "learning_rate": 5e-05, "loss": 0.05550145357847214, "num_tokens": 4165405.0, "reward": 0.06001953035593033, "reward_std": 0.6442891955375671, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.6599804759025574, "rewards/length_penalty/std": 0.2448662668466568, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9917033910751343, "sampling/importance_sampling_ratio/min": 0.13077811896800995, "sampling/sampling_logp_difference/max": 2.0342531204223633, "sampling/sampling_logp_difference/mean": 0.016961120069026947, "step": 54, "step_time": 25.952488951617852 }, { "clip_ratio/high_max": 0.0001538177442853339, "clip_ratio/high_mean": 0.0001538177442853339, "clip_ratio/low_mean": 0.00022297865216387437, "clip_ratio/low_min": 0.00022297865216387437, "clip_ratio/region_mean": 0.00037679638917325063, "completions/clipped_ratio": 0.3199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 1980.0, "completions/mean_length": 1385.8800048828125, "completions/mean_terminated_length": 1074.2940673828125, "completions/min_length": 627.0, "completions/min_terminated_length": 627.0, "entropy": 0.24276642501354218, "epoch": 0.14945652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.022746616974473, "learning_rate": 5e-05, "loss": 0.030116060748696327, "num_tokens": 4237589.0, "reward": 0.0033007811289280653, "reward_std": 0.7090996503829956, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.6766992211341858, "rewards/length_penalty/std": 0.26390570402145386, "sampling/importance_sampling_ratio/max": 2.452061414718628, "sampling/importance_sampling_ratio/mean": 0.9914399981498718, "sampling/importance_sampling_ratio/min": 0.2078368216753006, "sampling/sampling_logp_difference/max": 1.5710020065307617, "sampling/sampling_logp_difference/mean": 0.018045837059617043, "step": 55, "step_time": 25.46731282910332 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1928.0, "completions/mean_length": 1123.5799560546875, "completions/mean_terminated_length": 892.4750366210938, "completions/min_length": 463.0, "completions/min_terminated_length": 463.0, "entropy": 0.2039140909910202, "epoch": 0.15217391304347827, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4296838.0, "reward": 0.05137695372104645, "reward_std": 0.6519737243652344, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.5486230254173279, "rewards/length_penalty/std": 0.2840317487716675, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9926187992095947, "sampling/importance_sampling_ratio/min": 0.17601381242275238, "sampling/sampling_logp_difference/max": 1.7371928691864014, "sampling/sampling_logp_difference/mean": 0.015973661094903946, "step": 56, "step_time": 24.901454641018063 }, { "clip_ratio/high_max": 0.0001095643405278679, "clip_ratio/high_mean": 0.0001095643405278679, "clip_ratio/low_mean": 1.1971746425842867e-05, "clip_ratio/low_min": 1.1971746425842867e-05, "clip_ratio/region_mean": 0.00012153608695371076, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 2024.0, "completions/mean_length": 1327.47998046875, "completions/mean_terminated_length": 1018.6857299804688, "completions/min_length": 586.0, "completions/min_terminated_length": 586.0, "entropy": 0.25874788761138917, "epoch": 0.15489130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.017116200178861618, "learning_rate": 5e-05, "loss": 0.018467269837856293, "num_tokens": 4366452.0, "reward": 0.05181640386581421, "reward_std": 0.7132846713066101, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.6481835842132568, "rewards/length_penalty/std": 0.28123220801353455, "sampling/importance_sampling_ratio/max": 2.5459227561950684, "sampling/importance_sampling_ratio/mean": 0.9906158447265625, "sampling/importance_sampling_ratio/min": 0.25372740626335144, "sampling/sampling_logp_difference/max": 1.3714947700500488, "sampling/sampling_logp_difference/mean": 0.018632929772138596, "step": 57, "step_time": 25.201919438084587 }, { "clip_ratio/high_max": 0.00023499288072343917, "clip_ratio/high_mean": 0.00023499288072343917, "clip_ratio/low_mean": 0.00019223368435632436, "clip_ratio/low_min": 0.00019223368435632436, "clip_ratio/region_mean": 0.00042722656507976353, "completions/clipped_ratio": 0.3999999761581421, "completions/max_length": 2048.0, "completions/max_terminated_length": 2024.0, "completions/mean_length": 1528.2799072265625, "completions/mean_terminated_length": 1181.800048828125, "completions/min_length": 615.0, "completions/min_terminated_length": 615.0, "entropy": 0.24808227717876435, "epoch": 0.15760869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.02000986412167549, "learning_rate": 5e-05, "loss": 0.0544009804725647, "num_tokens": 4446676.0, "reward": -0.14623045921325684, "reward_std": 0.7256869077682495, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.7462304830551147, "rewards/length_penalty/std": 0.273065447807312, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9911220073699951, "sampling/importance_sampling_ratio/min": 0.12361037731170654, "sampling/sampling_logp_difference/max": 2.090620756149292, "sampling/sampling_logp_difference/mean": 0.01826377771794796, "step": 58, "step_time": 26.683956359047443 }, { "clip_ratio/high_max": 0.00035759122984018176, "clip_ratio/high_mean": 0.00035759122984018176, "clip_ratio/low_mean": 0.000107723061228171, "clip_ratio/low_min": 0.000107723061228171, "clip_ratio/region_mean": 0.0004653142997995019, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 1920.0, "completions/mean_length": 1148.919921875, "completions/mean_terminated_length": 1002.55810546875, "completions/min_length": 523.0, "completions/min_terminated_length": 523.0, "entropy": 0.20510495007038115, "epoch": 0.16032608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.023031344637274742, "learning_rate": 5e-05, "loss": 0.06129471957683563, "num_tokens": 4506832.0, "reward": 0.27900388836860657, "reward_std": 0.5668867230415344, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.5609961152076721, "rewards/length_penalty/std": 0.24307475984096527, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.992264986038208, "sampling/importance_sampling_ratio/min": 0.11492495238780975, "sampling/sampling_logp_difference/max": 2.16347599029541, "sampling/sampling_logp_difference/mean": 0.01719575934112072, "step": 59, "step_time": 24.862069938099012 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1517.0, "completions/max_terminated_length": 1517.0, "completions/mean_length": 1090.760009765625, "completions/mean_terminated_length": 1090.760009765625, "completions/min_length": 573.0, "completions/min_terminated_length": 573.0, "entropy": 0.16707909107208252, "epoch": 0.16304347826086957, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4564180.0, "reward": -0.1325976550579071, "reward_std": 0.5134265422821045, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.5325976610183716, "rewards/length_penalty/std": 0.1358153074979782, "sampling/importance_sampling_ratio/max": 2.3784713745117188, "sampling/importance_sampling_ratio/mean": 0.9938706755638123, "sampling/importance_sampling_ratio/min": 0.2581333816051483, "sampling/sampling_logp_difference/max": 1.354278802871704, "sampling/sampling_logp_difference/mean": 0.014030194841325283, "step": 60, "step_time": 18.95257089100778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1565.0, "completions/max_terminated_length": 1565.0, "completions/mean_length": 973.5, "completions/mean_terminated_length": 973.5, "completions/min_length": 561.0, "completions/min_terminated_length": 561.0, "entropy": 0.17081426084041595, "epoch": 0.16576086956521738, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4615265.0, "reward": 0.524658203125, "reward_std": 0.10992605239152908, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.475341796875, "rewards/length_penalty/std": 0.10992605239152908, "sampling/importance_sampling_ratio/max": 2.012568235397339, "sampling/importance_sampling_ratio/mean": 0.9934387803077698, "sampling/importance_sampling_ratio/min": 0.1067105382680893, "sampling/sampling_logp_difference/max": 2.237635374069214, "sampling/sampling_logp_difference/mean": 0.015026817098259926, "step": 61, "step_time": 18.586309609003365 }, { "clip_ratio/high_max": 0.0010058300336822867, "clip_ratio/high_mean": 0.0010058300336822867, "clip_ratio/low_mean": 7.036851311568171e-05, "clip_ratio/low_min": 7.036851311568171e-05, "clip_ratio/region_mean": 0.0010761985322460532, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 2016.0, "completions/mean_length": 1377.0599365234375, "completions/mean_terminated_length": 1209.3250732421875, "completions/min_length": 518.0, "completions/min_terminated_length": 518.0, "entropy": 0.18962046802043914, "epoch": 0.16847826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.020156601443886757, "learning_rate": 5e-05, "loss": 0.0698838084936142, "num_tokens": 4687738.0, "reward": 0.08760742098093033, "reward_std": 0.622724175453186, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.6723926067352295, "rewards/length_penalty/std": 0.24595586955547333, "sampling/importance_sampling_ratio/max": 2.3937716484069824, "sampling/importance_sampling_ratio/mean": 0.9929559826850891, "sampling/importance_sampling_ratio/min": 0.1018500030040741, "sampling/sampling_logp_difference/max": 2.2842540740966797, "sampling/sampling_logp_difference/mean": 0.015013793483376503, "step": 62, "step_time": 26.03344288491644 }, { "clip_ratio/high_max": 0.00013667871244251728, "clip_ratio/high_mean": 0.00013667871244251728, "clip_ratio/low_mean": 1.906941324705258e-05, "clip_ratio/low_min": 1.906941324705258e-05, "clip_ratio/region_mean": 0.00015574812568956985, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 1833.0, "completions/mean_length": 1001.8999633789062, "completions/mean_terminated_length": 935.1276245117188, "completions/min_length": 437.0, "completions/min_terminated_length": 437.0, "entropy": 0.1805823862552643, "epoch": 0.17119565217391305, "frac_reward_zero_std": 0.0, "grad_norm": 0.020034167915582657, "learning_rate": 5e-05, "loss": -0.026696156710386276, "num_tokens": 4741083.0, "reward": 0.33079099655151367, "reward_std": 0.5535070896148682, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.48920899629592896, "rewards/length_penalty/std": 0.20396488904953003, "sampling/importance_sampling_ratio/max": 2.1557698249816895, "sampling/importance_sampling_ratio/mean": 0.9933685064315796, "sampling/importance_sampling_ratio/min": 0.24905341863632202, "sampling/sampling_logp_difference/max": 1.3900878429412842, "sampling/sampling_logp_difference/mean": 0.015327067114412785, "step": 63, "step_time": 23.882240040693432 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1856.0, "completions/max_terminated_length": 1856.0, "completions/mean_length": 1082.3199462890625, "completions/mean_terminated_length": 1082.3199462890625, "completions/min_length": 648.0, "completions/min_terminated_length": 648.0, "entropy": 0.1634598731994629, "epoch": 0.17391304347826086, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4798719.0, "reward": 0.4715234339237213, "reward_std": 0.12802571058273315, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.5284765362739563, "rewards/length_penalty/std": 0.12802571058273315, "sampling/importance_sampling_ratio/max": 2.787916898727417, "sampling/importance_sampling_ratio/mean": 0.9940073490142822, "sampling/importance_sampling_ratio/min": 0.21132972836494446, "sampling/sampling_logp_difference/max": 1.554335594177246, "sampling/sampling_logp_difference/mean": 0.01441193837672472, "step": 64, "step_time": 22.091740830102935 }, { "clip_ratio/high_max": 0.00034654540359042586, "clip_ratio/high_mean": 0.00034654540359042586, "clip_ratio/low_mean": 7.996566564543173e-05, "clip_ratio/low_min": 7.996566564543173e-05, "clip_ratio/region_mean": 0.0004265110794221982, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1738.0, "completions/mean_length": 1163.0799560546875, "completions/mean_terminated_length": 1042.4091796875, "completions/min_length": 545.0, "completions/min_terminated_length": 545.0, "entropy": 0.20477714240550995, "epoch": 0.1766304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.01865788735449314, "learning_rate": 5e-05, "loss": 0.0512714609503746, "num_tokens": 4860583.0, "reward": 0.2920898497104645, "reward_std": 0.5272973775863647, "rewards/correctness/mean": 0.8600000143051147, "rewards/correctness/std": 0.3505098223686218, "rewards/length_penalty/mean": -0.5679101347923279, "rewards/length_penalty/std": 0.2127053588628769, "sampling/importance_sampling_ratio/max": 2.11232328414917, "sampling/importance_sampling_ratio/mean": 0.9926192760467529, "sampling/importance_sampling_ratio/min": 0.13898611068725586, "sampling/sampling_logp_difference/max": 1.9733812808990479, "sampling/sampling_logp_difference/mean": 0.01637178100645542, "step": 65, "step_time": 25.118174897041172 }, { "clip_ratio/high_max": 0.0019938923884183168, "clip_ratio/high_mean": 0.0019938923884183168, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019938923884183168, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1531.0, "completions/mean_length": 1054.1600341796875, "completions/mean_terminated_length": 805.7000122070312, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "entropy": 0.27262707352638244, "epoch": 0.1793478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.016474498435854912, "learning_rate": 5e-05, "loss": -0.019765282049775124, "num_tokens": 4916341.0, "reward": 0.2452734261751175, "reward_std": 0.6758597493171692, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.5147265791893005, "rewards/length_penalty/std": 0.2739177644252777, "sampling/importance_sampling_ratio/max": 2.592073678970337, "sampling/importance_sampling_ratio/mean": 0.9898561239242554, "sampling/importance_sampling_ratio/min": 0.14704741537570953, "sampling/sampling_logp_difference/max": 1.9170001745224, "sampling/sampling_logp_difference/mean": 0.02111000381410122, "step": 66, "step_time": 24.196540362667292 }, { "clip_ratio/high_max": 0.00013497067266143858, "clip_ratio/high_mean": 0.00013497067266143858, "clip_ratio/low_mean": 7.393147825496271e-05, "clip_ratio/low_min": 7.393147825496271e-05, "clip_ratio/region_mean": 0.00020890214655082672, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1807.0, "completions/mean_length": 1059.3399658203125, "completions/mean_terminated_length": 1039.1632080078125, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.14760487377643586, "epoch": 0.18206521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.026212945580482483, "learning_rate": 5e-05, "loss": 0.06858495622873306, "num_tokens": 4972258.0, "reward": 0.062744140625, "reward_std": 0.5660516619682312, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.4985693693161011, "rewards/length_penalty/mean": -0.5172558426856995, "rewards/length_penalty/std": 0.18850365281105042, "sampling/importance_sampling_ratio/max": 2.830606698989868, "sampling/importance_sampling_ratio/mean": 0.9945699572563171, "sampling/importance_sampling_ratio/min": 0.16257956624031067, "sampling/sampling_logp_difference/max": 1.8165876865386963, "sampling/sampling_logp_difference/mean": 0.012930328026413918, "step": 67, "step_time": 24.303856736049056 }, { "clip_ratio/high_max": 0.00043857565469807014, "clip_ratio/high_mean": 0.00043857565469807014, "clip_ratio/low_mean": 0.00010617515217745677, "clip_ratio/low_min": 0.00010617515217745677, "clip_ratio/region_mean": 0.0005447508214274421, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1929.0, "completions/mean_length": 1126.02001953125, "completions/mean_terminated_length": 1087.604248046875, "completions/min_length": 688.0, "completions/min_terminated_length": 688.0, "entropy": 0.15897513329982757, "epoch": 0.18478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.01748628169298172, "learning_rate": 5e-05, "loss": 0.008149566128849983, "num_tokens": 5032319.0, "reward": 0.23018553853034973, "reward_std": 0.5640369653701782, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.5498144626617432, "rewards/length_penalty/std": 0.18217217922210693, "sampling/importance_sampling_ratio/max": 2.4909420013427734, "sampling/importance_sampling_ratio/mean": 0.9942116141319275, "sampling/importance_sampling_ratio/min": 0.2051500827074051, "sampling/sampling_logp_difference/max": 1.5840134620666504, "sampling/sampling_logp_difference/mean": 0.013285518623888493, "step": 68, "step_time": 24.305004102177918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1440.0, "completions/mean_length": 1111.4599609375, "completions/mean_terminated_length": 877.3250122070312, "completions/min_length": 484.0, "completions/min_terminated_length": 484.0, "entropy": 0.2204485982656479, "epoch": 0.1875, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5090272.0, "reward": 0.05729492008686066, "reward_std": 0.657320499420166, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.5427050590515137, "rewards/length_penalty/std": 0.25102466344833374, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9920188188552856, "sampling/importance_sampling_ratio/min": 0.1841098666191101, "sampling/sampling_logp_difference/max": 1.6922225952148438, "sampling/sampling_logp_difference/mean": 0.017389731481671333, "step": 69, "step_time": 24.247353052021936 }, { "clip_ratio/high_max": 3.230756847187877e-05, "clip_ratio/high_mean": 3.230756847187877e-05, "clip_ratio/low_mean": 7.781875465298071e-05, "clip_ratio/low_min": 7.781875465298071e-05, "clip_ratio/region_mean": 0.00011012632312485948, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 1924.0, "completions/mean_length": 1292.52001953125, "completions/mean_terminated_length": 998.7222290039062, "completions/min_length": 572.0, "completions/min_terminated_length": 572.0, "entropy": 0.15242201685905457, "epoch": 0.19021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.021190080791711807, "learning_rate": 5e-05, "loss": -0.013169449754059315, "num_tokens": 5158408.0, "reward": -0.41111326217651367, "reward_std": 0.5726756453514099, "rewards/correctness/mean": 0.2199999988079071, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.6311132907867432, "rewards/length_penalty/std": 0.28315505385398865, "sampling/importance_sampling_ratio/max": 2.3609304428100586, "sampling/importance_sampling_ratio/mean": 0.9941762089729309, "sampling/importance_sampling_ratio/min": 0.1941743642091751, "sampling/sampling_logp_difference/max": 1.6389987468719482, "sampling/sampling_logp_difference/mean": 0.012777176685631275, "step": 70, "step_time": 25.66260247514583 }, { "clip_ratio/high_max": 0.00031170886504696684, "clip_ratio/high_mean": 0.00031170886504696684, "clip_ratio/low_mean": 0.0002567291259765625, "clip_ratio/low_min": 0.0002567291259765625, "clip_ratio/region_mean": 0.0005684379924787208, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 1691.0, "completions/mean_length": 970.97998046875, "completions/mean_terminated_length": 902.2340087890625, "completions/min_length": 521.0, "completions/min_terminated_length": 521.0, "entropy": 0.2098962813615799, "epoch": 0.19293478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.018765395507216454, "learning_rate": 5e-05, "loss": -0.005327221006155014, "num_tokens": 5209317.0, "reward": 0.10588867217302322, "reward_std": 0.566224217414856, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.4985693693161011, "rewards/length_penalty/mean": -0.47411131858825684, "rewards/length_penalty/std": 0.18729694187641144, "sampling/importance_sampling_ratio/max": 2.3629136085510254, "sampling/importance_sampling_ratio/mean": 0.9917638301849365, "sampling/importance_sampling_ratio/min": 0.1295989453792572, "sampling/sampling_logp_difference/max": 2.0433106422424316, "sampling/sampling_logp_difference/mean": 0.018866313621401787, "step": 71, "step_time": 23.478281326359138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1741.0, "completions/max_terminated_length": 1741.0, "completions/mean_length": 898.6399536132812, "completions/mean_terminated_length": 898.6399536132812, "completions/min_length": 632.0, "completions/min_terminated_length": 632.0, "entropy": 0.17583955228328704, "epoch": 0.1956521739130435, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5257919.0, "reward": 0.3612109422683716, "reward_std": 0.4679783582687378, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.4387890696525574, "rewards/length_penalty/std": 0.10696899890899658, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9934483766555786, "sampling/importance_sampling_ratio/min": 0.14307568967342377, "sampling/sampling_logp_difference/max": 1.9443814754486084, "sampling/sampling_logp_difference/mean": 0.015687277540564537, "step": 72, "step_time": 20.240301510784775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1322.0, "completions/mean_length": 1160.47998046875, "completions/mean_terminated_length": 938.6000366210938, "completions/min_length": 620.0, "completions/min_terminated_length": 620.0, "entropy": 0.20923268496990205, "epoch": 0.1983695652173913, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5318353.0, "reward": 0.03335937485098839, "reward_std": 0.6578137278556824, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.5666406154632568, "rewards/length_penalty/std": 0.23294444382190704, "sampling/importance_sampling_ratio/max": 2.6613593101501465, "sampling/importance_sampling_ratio/mean": 0.9921817183494568, "sampling/importance_sampling_ratio/min": 0.17200525104999542, "sampling/sampling_logp_difference/max": 1.760230302810669, "sampling/sampling_logp_difference/mean": 0.016782034188508987, "step": 73, "step_time": 24.930544017115608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 1819.0, "completions/mean_length": 1072.739990234375, "completions/mean_terminated_length": 886.9761962890625, "completions/min_length": 541.0, "completions/min_terminated_length": 541.0, "entropy": 0.25260456204414367, "epoch": 0.20108695652173914, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5375320.0, "reward": -0.12379882484674454, "reward_std": 0.6523011922836304, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.5237988233566284, "rewards/length_penalty/std": 0.24123379588127136, "sampling/importance_sampling_ratio/max": 2.323164224624634, "sampling/importance_sampling_ratio/mean": 0.990655779838562, "sampling/importance_sampling_ratio/min": 0.12009541690349579, "sampling/sampling_logp_difference/max": 2.1194686889648438, "sampling/sampling_logp_difference/mean": 0.02063530683517456, "step": 74, "step_time": 24.19013630389236 }, { "clip_ratio/high_max": 0.0011373767745681107, "clip_ratio/high_mean": 0.0011373767745681107, "clip_ratio/low_mean": 0.00011958675750065595, "clip_ratio/low_min": 0.00011958675750065595, "clip_ratio/region_mean": 0.0012569635291583836, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 2014.0, "completions/mean_length": 1109.4000244140625, "completions/mean_terminated_length": 1049.4893798828125, "completions/min_length": 503.0, "completions/min_terminated_length": 503.0, "entropy": 0.2316681683063507, "epoch": 0.20380434782608695, "frac_reward_zero_std": 0.0, "grad_norm": 0.023913364857435226, "learning_rate": 5e-05, "loss": 0.02908468060195446, "num_tokens": 5433120.0, "reward": 0.3183007836341858, "reward_std": 0.49624332785606384, "rewards/correctness/mean": 0.8600000143051147, "rewards/correctness/std": 0.3505098223686218, "rewards/length_penalty/mean": -0.541699230670929, "rewards/length_penalty/std": 0.23889291286468506, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9917033314704895, "sampling/importance_sampling_ratio/min": 0.08821981400251389, "sampling/sampling_logp_difference/max": 2.4279236793518066, "sampling/sampling_logp_difference/mean": 0.019385503605008125, "step": 75, "step_time": 24.24726929096505 }, { "clip_ratio/high_max": 0.00020537180680548772, "clip_ratio/high_mean": 0.00020537180680548772, "clip_ratio/low_mean": 0.00020198143174638973, "clip_ratio/low_min": 0.00020198143174638973, "clip_ratio/region_mean": 0.0004073532385518774, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 1748.0, "completions/mean_length": 1043.93994140625, "completions/mean_terminated_length": 852.6904907226562, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.24002922773361207, "epoch": 0.20652173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.016545172780752182, "learning_rate": 5e-05, "loss": 0.007756037637591362, "num_tokens": 5488657.0, "reward": 0.270263671875, "reward_std": 0.6482783555984497, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.5097362995147705, "rewards/length_penalty/std": 0.25542691349983215, "sampling/importance_sampling_ratio/max": 1.7395713329315186, "sampling/importance_sampling_ratio/mean": 0.9911350607872009, "sampling/importance_sampling_ratio/min": 0.21664908528327942, "sampling/sampling_logp_difference/max": 1.5294764041900635, "sampling/sampling_logp_difference/mean": 0.019268445670604706, "step": 76, "step_time": 24.103159786434844 }, { "clip_ratio/high_max": 0.00022875646827742458, "clip_ratio/high_mean": 0.00022875646827742458, "clip_ratio/low_mean": 0.00020549298205878584, "clip_ratio/low_min": 0.00020549298205878584, "clip_ratio/region_mean": 0.0004342494561569765, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 2043.0, "completions/mean_length": 1050.52001953125, "completions/mean_terminated_length": 860.5238037109375, "completions/min_length": 461.0, "completions/min_terminated_length": 461.0, "entropy": 0.21642525792121886, "epoch": 0.20923913043478262, "frac_reward_zero_std": 0.0, "grad_norm": 0.022253649309277534, "learning_rate": 5e-05, "loss": 0.029880687594413757, "num_tokens": 5543743.0, "reward": 0.3270507752895355, "reward_std": 0.6117151975631714, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.5129492282867432, "rewards/length_penalty/std": 0.27931278944015503, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9922550320625305, "sampling/importance_sampling_ratio/min": 0.18467582762241364, "sampling/sampling_logp_difference/max": 1.6891531944274902, "sampling/sampling_logp_difference/mean": 0.01771993562579155, "step": 77, "step_time": 24.59413585718721 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 1986.0, "completions/mean_length": 1048.679931640625, "completions/mean_terminated_length": 961.7826538085938, "completions/min_length": 534.0, "completions/min_terminated_length": 534.0, "entropy": 0.16875274777412413, "epoch": 0.21195652173913043, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5598477.0, "reward": 0.08794921636581421, "reward_std": 0.6423997282981873, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.5120508074760437, "rewards/length_penalty/std": 0.20729389786720276, "sampling/importance_sampling_ratio/max": 1.9575339555740356, "sampling/importance_sampling_ratio/mean": 0.9939072132110596, "sampling/importance_sampling_ratio/min": 0.18968318402767181, "sampling/sampling_logp_difference/max": 1.6624000072479248, "sampling/sampling_logp_difference/mean": 0.015094350092113018, "step": 78, "step_time": 23.937921254662797 }, { "clip_ratio/high_max": 0.00026702526811277493, "clip_ratio/high_mean": 0.00026702526811277493, "clip_ratio/low_mean": 0.00010438936587888747, "clip_ratio/low_min": 0.00010438936587888747, "clip_ratio/region_mean": 0.00037141462526051327, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1726.0, "completions/mean_length": 1113.1600341796875, "completions/mean_terminated_length": 1094.0816650390625, "completions/min_length": 603.0, "completions/min_terminated_length": 603.0, "entropy": 0.15124431550502776, "epoch": 0.21467391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.022289307788014412, "learning_rate": 5e-05, "loss": 0.04329625517129898, "num_tokens": 5657595.0, "reward": 0.23646484315395355, "reward_std": 0.44142961502075195, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.5435351729393005, "rewards/length_penalty/std": 0.1726207286119461, "sampling/importance_sampling_ratio/max": 2.301612615585327, "sampling/importance_sampling_ratio/mean": 0.9943583011627197, "sampling/importance_sampling_ratio/min": 0.10937031358480453, "sampling/sampling_logp_difference/max": 2.2130157947540283, "sampling/sampling_logp_difference/mean": 0.013710588216781616, "step": 79, "step_time": 24.704213061137125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1857.0, "completions/mean_length": 1002.4599609375, "completions/mean_terminated_length": 981.1224365234375, "completions/min_length": 649.0, "completions/min_terminated_length": 649.0, "entropy": 0.18383682072162627, "epoch": 0.21739130434782608, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5710878.0, "reward": -0.08948241919279099, "reward_std": 0.5880789756774902, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.48948243260383606, "rewards/length_penalty/std": 0.13900429010391235, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.993415117263794, "sampling/importance_sampling_ratio/min": 0.1942727416753769, "sampling/sampling_logp_difference/max": 1.638492226600647, "sampling/sampling_logp_difference/mean": 0.015673737972974777, "step": 80, "step_time": 23.73240554286167 }, { "clip_ratio/high_max": 0.00033401059918105604, "clip_ratio/high_mean": 0.00033401059918105604, "clip_ratio/low_mean": 9.887570195132867e-05, "clip_ratio/low_min": 9.887570195132867e-05, "clip_ratio/region_mean": 0.00043288629676681013, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1887.0, "completions/mean_length": 1197.3399658203125, "completions/mean_terminated_length": 1161.8958740234375, "completions/min_length": 519.0, "completions/min_terminated_length": 519.0, "entropy": 0.17664674520492554, "epoch": 0.22010869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.023780491203069687, "learning_rate": 5e-05, "loss": 0.0315287783741951, "num_tokens": 5775205.0, "reward": 0.1553613245487213, "reward_std": 0.4048424959182739, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308748841285706, "rewards/length_penalty/mean": -0.5846386551856995, "rewards/length_penalty/std": 0.19399148225784302, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9934639930725098, "sampling/importance_sampling_ratio/min": 0.13834220170974731, "sampling/sampling_logp_difference/max": 1.978024959564209, "sampling/sampling_logp_difference/mean": 0.015248614363372326, "step": 81, "step_time": 25.163222548784688 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1238.0, "completions/max_terminated_length": 1238.0, "completions/mean_length": 817.739990234375, "completions/mean_terminated_length": 817.739990234375, "completions/min_length": 472.0, "completions/min_terminated_length": 472.0, "entropy": 0.17340467274188995, "epoch": 0.22282608695652173, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5818942.0, "reward": 0.40071287751197815, "reward_std": 0.4565727710723877, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.3992871046066284, "rewards/length_penalty/std": 0.09006626904010773, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9931971430778503, "sampling/importance_sampling_ratio/min": 0.11889954656362534, "sampling/sampling_logp_difference/max": 2.129476308822632, "sampling/sampling_logp_difference/mean": 0.01671450585126877, "step": 82, "step_time": 14.970575261162594 }, { "clip_ratio/high_max": 0.000732912658713758, "clip_ratio/high_mean": 0.000732912658713758, "clip_ratio/low_mean": 0.00014227035426301883, "clip_ratio/low_min": 0.00014227035426301883, "clip_ratio/region_mean": 0.0008751830086112022, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1687.0, "completions/mean_length": 941.0, "completions/mean_terminated_length": 918.4081420898438, "completions/min_length": 585.0, "completions/min_terminated_length": 585.0, "entropy": 0.19994757175445557, "epoch": 0.22554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.016098052263259888, "learning_rate": 5e-05, "loss": -0.004672223702073097, "num_tokens": 5868422.0, "reward": 0.1005273386836052, "reward_std": 0.5388616919517517, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.45947265625, "rewards/length_penalty/std": 0.1366553157567978, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9926522970199585, "sampling/importance_sampling_ratio/min": 0.1678345501422882, "sampling/sampling_logp_difference/max": 1.7847766876220703, "sampling/sampling_logp_difference/mean": 0.017932169139385223, "step": 83, "step_time": 23.696208319161087 }, { "clip_ratio/high_max": 0.0003068212536163628, "clip_ratio/high_mean": 0.0003068212536163628, "clip_ratio/low_mean": 6.011179066263139e-05, "clip_ratio/low_min": 6.011179066263139e-05, "clip_ratio/region_mean": 0.0003669330530101433, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 2036.0, "completions/mean_length": 955.6199951171875, "completions/mean_terminated_length": 910.1041870117188, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "entropy": 0.1619774729013443, "epoch": 0.22826086956521738, "frac_reward_zero_std": 0.0, "grad_norm": 0.017402462661266327, "learning_rate": 5e-05, "loss": 0.016482943668961525, "num_tokens": 5919183.0, "reward": 0.3133886754512787, "reward_std": 0.45723626017570496, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.4666113257408142, "rewards/length_penalty/std": 0.24670767784118652, "sampling/importance_sampling_ratio/max": 2.7439754009246826, "sampling/importance_sampling_ratio/mean": 0.9940432906150818, "sampling/importance_sampling_ratio/min": 0.20785371959209442, "sampling/sampling_logp_difference/max": 1.570920705795288, "sampling/sampling_logp_difference/mean": 0.015178903006017208, "step": 84, "step_time": 23.68598745414056 }, { "clip_ratio/high_max": 0.00014658061554655432, "clip_ratio/high_mean": 0.00014658061554655432, "clip_ratio/low_mean": 0.00022748662740923464, "clip_ratio/low_min": 0.00022748662740923464, "clip_ratio/region_mean": 0.00037406723422463985, "completions/clipped_ratio": 0.35999998450279236, "completions/max_length": 2048.0, "completions/max_terminated_length": 1877.0, "completions/mean_length": 1241.4599609375, "completions/mean_terminated_length": 787.78125, "completions/min_length": 437.0, "completions/min_terminated_length": 437.0, "entropy": 0.2703008711338043, "epoch": 0.23097826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.02500966563820839, "learning_rate": 5e-05, "loss": 0.02356114238500595, "num_tokens": 5983276.0, "reward": -0.16618163883686066, "reward_std": 0.7458595037460327, "rewards/correctness/mean": 0.4399999976158142, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.6061816215515137, "rewards/length_penalty/std": 0.32063478231430054, "sampling/importance_sampling_ratio/max": 2.443023443222046, "sampling/importance_sampling_ratio/mean": 0.9897676110267639, "sampling/importance_sampling_ratio/min": 0.1255943924188614, "sampling/sampling_logp_difference/max": 2.074697732925415, "sampling/sampling_logp_difference/mean": 0.021558986976742744, "step": 85, "step_time": 24.5623816922307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1246.0, "completions/max_terminated_length": 1246.0, "completions/mean_length": 681.1599731445312, "completions/mean_terminated_length": 681.1599731445312, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.17943477034568786, "epoch": 0.23369565217391305, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 6019434.0, "reward": 0.6674023270606995, "reward_std": 0.10376235842704773, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.33259764313697815, "rewards/length_penalty/std": 0.10376234352588654, "sampling/importance_sampling_ratio/max": 2.498547315597534, "sampling/importance_sampling_ratio/mean": 0.9930832982063293, "sampling/importance_sampling_ratio/min": 0.1694667786359787, "sampling/sampling_logp_difference/max": 1.7750983238220215, "sampling/sampling_logp_difference/mean": 0.017346223816275597, "step": 86, "step_time": 14.885744557948783 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1871.0, "completions/max_terminated_length": 1871.0, "completions/mean_length": 999.5199584960938, "completions/mean_terminated_length": 999.5199584960938, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.15669138133525848, "epoch": 0.23641304347826086, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 6071620.0, "reward": 0.11195312440395355, "reward_std": 0.6052372455596924, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.48804688453674316, "rewards/length_penalty/std": 0.1995229423046112, "sampling/importance_sampling_ratio/max": 2.6485512256622314, "sampling/importance_sampling_ratio/mean": 0.9941596388816833, "sampling/importance_sampling_ratio/min": 0.1565091609954834, "sampling/sampling_logp_difference/max": 1.8546407222747803, "sampling/sampling_logp_difference/mean": 0.014136388897895813, "step": 87, "step_time": 21.96357577503659 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1286.0, "completions/max_terminated_length": 1286.0, "completions/mean_length": 781.5999755859375, "completions/mean_terminated_length": 781.5999755859375, "completions/min_length": 453.0, "completions/min_terminated_length": 453.0, "entropy": 0.1509826362133026, "epoch": 0.2391304347826087, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 6113590.0, "reward": 0.618359386920929, "reward_std": 0.10561134666204453, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.38164061307907104, "rewards/length_penalty/std": 0.10561134666204453, "sampling/importance_sampling_ratio/max": 2.0460262298583984, "sampling/importance_sampling_ratio/mean": 0.9942871928215027, "sampling/importance_sampling_ratio/min": 0.17832712829113007, "sampling/sampling_logp_difference/max": 1.7241356372833252, "sampling/sampling_logp_difference/mean": 0.014685031026601791, "step": 88, "step_time": 15.273658250225708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.17999999225139618, "completions/max_length": 2048.0, "completions/max_terminated_length": 1500.0, "completions/mean_length": 1074.1199951171875, "completions/mean_terminated_length": 860.3414306640625, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "entropy": 0.1885121375322342, "epoch": 0.2418478260869565, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 6170756.0, "reward": 0.07552734017372131, "reward_std": 0.6841561198234558, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.5244726538658142, "rewards/length_penalty/std": 0.2534148693084717, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9928888082504272, "sampling/importance_sampling_ratio/min": 0.21019017696380615, "sampling/sampling_logp_difference/max": 1.9624786376953125, "sampling/sampling_logp_difference/mean": 0.016248032450675964, "step": 89, "step_time": 24.75006088986993 }, { "clip_ratio/high_max": 0.0008607835246948525, "clip_ratio/high_mean": 0.0008607835246948525, "clip_ratio/low_mean": 7.468502008123323e-05, "clip_ratio/low_min": 7.468502008123323e-05, "clip_ratio/region_mean": 0.000935468528768979, "completions/clipped_ratio": 0.23999999463558197, "completions/max_length": 2048.0, "completions/max_terminated_length": 2025.0, "completions/mean_length": 1286.43994140625, "completions/mean_terminated_length": 1045.9473876953125, "completions/min_length": 569.0, "completions/min_terminated_length": 569.0, "entropy": 0.15907079875469207, "epoch": 0.24456521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.017984451726078987, "learning_rate": 5e-05, "loss": 0.053571514785289764, "num_tokens": 6238108.0, "reward": 0.11185546964406967, "reward_std": 0.6685841083526611, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.6281445026397705, "rewards/length_penalty/std": 0.262511670589447, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9940212965011597, "sampling/importance_sampling_ratio/min": 0.11006206274032593, "sampling/sampling_logp_difference/max": 2.2067108154296875, "sampling/sampling_logp_difference/mean": 0.013727168552577496, "step": 90, "step_time": 24.970630661817268 }, { "clip_ratio/high_max": 2.9875271138735116e-05, "clip_ratio/high_mean": 2.9875271138735116e-05, "clip_ratio/low_mean": 7.273093942785635e-05, "clip_ratio/low_min": 7.273093942785635e-05, "clip_ratio/region_mean": 0.00010260620911139995, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1772.0, "completions/mean_length": 1100.93994140625, "completions/mean_terminated_length": 864.1749877929688, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.14755462408065795, "epoch": 0.24728260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.01865803636610508, "learning_rate": 5e-05, "loss": 0.00030498579144477844, "num_tokens": 6298325.0, "reward": 0.28243163228034973, "reward_std": 0.6337281465530396, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.5375683307647705, "rewards/length_penalty/std": 0.28482669591903687, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9942618608474731, "sampling/importance_sampling_ratio/min": 0.11354823410511017, "sampling/sampling_logp_difference/max": 2.175527572631836, "sampling/sampling_logp_difference/mean": 0.01375518273562193, "step": 91, "step_time": 25.209162486949936 }, { "clip_ratio/high_max": 7.004640647210181e-05, "clip_ratio/high_mean": 7.004640647210181e-05, "clip_ratio/low_mean": 1.4971180644351988e-05, "clip_ratio/low_min": 1.4971180644351988e-05, "clip_ratio/region_mean": 8.501758711645379e-05, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1354.0, "completions/mean_length": 1052.0599365234375, "completions/mean_terminated_length": 803.0750122070312, "completions/min_length": 456.0, "completions/min_terminated_length": 456.0, "entropy": 0.18619080781936645, "epoch": 0.25, "frac_reward_zero_std": 0.0, "grad_norm": 0.01825055480003357, "learning_rate": 5e-05, "loss": -0.00015388429164886475, "num_tokens": 6353318.0, "reward": 0.10629882663488388, "reward_std": 0.6727583408355713, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.5137012004852295, "rewards/length_penalty/std": 0.26483675837516785, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930269122123718, "sampling/importance_sampling_ratio/min": 0.09843255579471588, "sampling/sampling_logp_difference/max": 2.3183836936950684, "sampling/sampling_logp_difference/mean": 0.016253095120191574, "step": 92, "step_time": 24.55524198617786 }, { "clip_ratio/high_max": 8.259689784608781e-05, "clip_ratio/high_mean": 8.259689784608781e-05, "clip_ratio/low_mean": 0.0001133478173869662, "clip_ratio/low_min": 0.0001133478173869662, "clip_ratio/region_mean": 0.00019594471377786248, "completions/clipped_ratio": 0.3199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 2013.0, "completions/mean_length": 1414.159912109375, "completions/mean_terminated_length": 1115.88232421875, "completions/min_length": 650.0, "completions/min_terminated_length": 650.0, "entropy": 0.1744084507226944, "epoch": 0.25271739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.01962885633111, "learning_rate": 5e-05, "loss": 0.007613500580191612, "num_tokens": 6429646.0, "reward": -0.010507812723517418, "reward_std": 0.6971383094787598, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.4712120592594147, "rewards/length_penalty/mean": -0.6905078291893005, "rewards/length_penalty/std": 0.2682313621044159, "sampling/importance_sampling_ratio/max": 1.75165855884552, "sampling/importance_sampling_ratio/mean": 0.9932826161384583, "sampling/importance_sampling_ratio/min": 0.13028010725975037, "sampling/sampling_logp_difference/max": 2.0380685329437256, "sampling/sampling_logp_difference/mean": 0.014411170966923237, "step": 93, "step_time": 26.294387294212356 }, { "clip_ratio/high_max": 0.0012861603521741928, "clip_ratio/high_mean": 0.0012861603521741928, "clip_ratio/low_mean": 1.874238660093397e-05, "clip_ratio/low_min": 1.874238660093397e-05, "clip_ratio/region_mean": 0.0013049027300439775, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1686.0, "completions/mean_length": 1055.6400146484375, "completions/mean_terminated_length": 1014.2916870117188, "completions/min_length": 579.0, "completions/min_terminated_length": 579.0, "entropy": 0.18473588824272155, "epoch": 0.2554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.021314339712262154, "learning_rate": 5e-05, "loss": 0.013898667879402637, "num_tokens": 6488458.0, "reward": 0.044550780206918716, "reward_std": 0.45703670382499695, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.5154492259025574, "rewards/length_penalty/std": 0.18456950783729553, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930935502052307, "sampling/importance_sampling_ratio/min": 0.22412633895874023, "sampling/sampling_logp_difference/max": 1.4955453872680664, "sampling/sampling_logp_difference/mean": 0.016116132959723473, "step": 94, "step_time": 25.04625384393148 }, { "clip_ratio/high_max": 0.0012738024466671049, "clip_ratio/high_mean": 0.0012738024466671049, "clip_ratio/low_mean": 1.8919685680884867e-05, "clip_ratio/low_min": 1.8919685680884867e-05, "clip_ratio/region_mean": 0.0012927221367135644, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1629.0, "completions/mean_length": 890.5, "completions/mean_terminated_length": 842.2708740234375, "completions/min_length": 403.0, "completions/min_terminated_length": 403.0, "entropy": 0.19709089398384094, "epoch": 0.25815217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.02763357199728489, "learning_rate": 5e-05, "loss": 0.10320558398962021, "num_tokens": 6535353.0, "reward": 0.5251855254173279, "reward_std": 0.34238356351852417, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.434814453125, "rewards/length_penalty/std": 0.17862030863761902, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.992729902267456, "sampling/importance_sampling_ratio/min": 0.14601276814937592, "sampling/sampling_logp_difference/max": 1.9240611791610718, "sampling/sampling_logp_difference/mean": 0.017498843371868134, "step": 95, "step_time": 23.661474538501352 }, { "clip_ratio/high_max": 0.00022887169470777735, "clip_ratio/high_mean": 0.00022887169470777735, "clip_ratio/low_mean": 0.00011928312014788389, "clip_ratio/low_min": 0.00011928312014788389, "clip_ratio/region_mean": 0.00034815480903489516, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 1745.0, "completions/mean_length": 1020.7599487304688, "completions/mean_terminated_length": 955.1914672851562, "completions/min_length": 608.0, "completions/min_terminated_length": 608.0, "entropy": 0.18319753408432007, "epoch": 0.2608695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.017121320590376854, "learning_rate": 5e-05, "loss": -0.0056983958929777145, "num_tokens": 6589921.0, "reward": 0.2815820276737213, "reward_std": 0.5517584085464478, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.4984179735183716, "rewards/length_penalty/std": 0.175412118434906, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9933992028236389, "sampling/importance_sampling_ratio/min": 0.11078064888715744, "sampling/sampling_logp_difference/max": 2.2002031803131104, "sampling/sampling_logp_difference/mean": 0.01637713424861431, "step": 96, "step_time": 23.875369309913367 }, { "clip_ratio/high_max": 0.000689817895181477, "clip_ratio/high_mean": 0.000689817895181477, "clip_ratio/low_mean": 6.741062679793686e-05, "clip_ratio/low_min": 6.741062679793686e-05, "clip_ratio/region_mean": 0.0007572285190690309, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1928.0, "completions/mean_length": 957.47998046875, "completions/mean_terminated_length": 912.0416870117188, "completions/min_length": 499.0, "completions/min_terminated_length": 499.0, "entropy": 0.19041045010089874, "epoch": 0.26358695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.02426239848136902, "learning_rate": 5e-05, "loss": 0.09774880111217499, "num_tokens": 6640725.0, "reward": 0.472480446100235, "reward_std": 0.39451903104782104, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979514837265, "rewards/length_penalty/mean": -0.46751952171325684, "rewards/length_penalty/std": 0.18895146250724792, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930346012115479, "sampling/importance_sampling_ratio/min": 0.08964910358190536, "sampling/sampling_logp_difference/max": 2.4118521213531494, "sampling/sampling_logp_difference/mean": 0.01707942970097065, "step": 97, "step_time": 23.533656641840935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 1863.0, "completions/mean_length": 879.8399658203125, "completions/mean_terminated_length": 657.3333740234375, "completions/min_length": 396.0, "completions/min_terminated_length": 396.0, "entropy": 0.22813548147678375, "epoch": 0.266304347826087, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 6686857.0, "reward": -0.029609374701976776, "reward_std": 0.6830533742904663, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.42960938811302185, "rewards/length_penalty/std": 0.2751394212245941, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9917917251586914, "sampling/importance_sampling_ratio/min": 0.104432612657547, "sampling/sampling_logp_difference/max": 2.2592132091522217, "sampling/sampling_logp_difference/mean": 0.019895542412996292, "step": 98, "step_time": 23.863430125406012 }, { "clip_ratio/high_max": 0.0001910436199977994, "clip_ratio/high_mean": 0.0001910436199977994, "clip_ratio/low_mean": 0.00016743242740631105, "clip_ratio/low_min": 0.00016743242740631105, "clip_ratio/region_mean": 0.00035847605322487653, "completions/clipped_ratio": 0.0, "completions/max_length": 1066.0, "completions/max_terminated_length": 1066.0, "completions/mean_length": 727.719970703125, "completions/mean_terminated_length": 727.719970703125, "completions/min_length": 457.0, "completions/min_terminated_length": 457.0, "entropy": 0.18468935191631317, "epoch": 0.26902173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.016533832997083664, "learning_rate": 5e-05, "loss": -0.01166108250617981, "num_tokens": 6726153.0, "reward": 0.3846679627895355, "reward_std": 0.4649888873100281, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.35533201694488525, "rewards/length_penalty/std": 0.07406768202781677, "sampling/importance_sampling_ratio/max": 2.5298891067504883, "sampling/importance_sampling_ratio/mean": 0.9932071566581726, "sampling/importance_sampling_ratio/min": 0.13677681982517242, "sampling/sampling_logp_difference/max": 1.9894046783447266, "sampling/sampling_logp_difference/mean": 0.01752503030002117, "step": 99, "step_time": 13.03185867331922 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1031.0, "completions/max_terminated_length": 1031.0, "completions/mean_length": 699.9599609375, "completions/mean_terminated_length": 699.9599609375, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "entropy": 0.17614607512950897, "epoch": 0.2717391304347826, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 6763381.0, "reward": 0.6582226157188416, "reward_std": 0.08882923424243927, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.34177735447883606, "rewards/length_penalty/std": 0.08882923424243927, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9934257864952087, "sampling/importance_sampling_ratio/min": 0.0938405692577362, "sampling/sampling_logp_difference/max": 2.3661580085754395, "sampling/sampling_logp_difference/mean": 0.01747218705713749, "step": 100, "step_time": 12.429255658062175 }, { "clip_ratio/high_max": 0.00019218685483792796, "clip_ratio/high_mean": 0.00019218685483792796, "clip_ratio/low_mean": 0.00023746510851196945, "clip_ratio/low_min": 0.00023746510851196945, "clip_ratio/region_mean": 0.00042965196771547196, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1399.0, "completions/mean_length": 927.719970703125, "completions/mean_terminated_length": 904.8571166992188, "completions/min_length": 570.0, "completions/min_terminated_length": 570.0, "entropy": 0.17351745665073395, "epoch": 0.27445652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.027100425213575363, "learning_rate": 5e-05, "loss": 0.05554354935884476, "num_tokens": 6812657.0, "reward": 0.5270116925239563, "reward_std": 0.2404518723487854, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.45298826694488525, "rewards/length_penalty/std": 0.12445920705795288, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9934037923812866, "sampling/importance_sampling_ratio/min": 0.0930493101477623, "sampling/sampling_logp_difference/max": 2.3746256828308105, "sampling/sampling_logp_difference/mean": 0.016070544719696045, "step": 101, "step_time": 23.550993845099583 }, { "clip_ratio/high_max": 0.0012817931477911771, "clip_ratio/high_mean": 0.0012817931477911771, "clip_ratio/low_mean": 1.8358729721512646e-05, "clip_ratio/low_min": 1.8358729721512646e-05, "clip_ratio/region_mean": 0.0013001518789678812, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1662.0, "completions/mean_length": 938.4199829101562, "completions/mean_terminated_length": 915.7755126953125, "completions/min_length": 421.0, "completions/min_terminated_length": 421.0, "entropy": 0.18813581466674806, "epoch": 0.27717391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.023955991491675377, "learning_rate": 5e-05, "loss": 0.024088561534881592, "num_tokens": 6862508.0, "reward": 0.5017871260643005, "reward_std": 0.3250575065612793, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.45821288228034973, "rewards/length_penalty/std": 0.1816956251859665, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.992979109287262, "sampling/importance_sampling_ratio/min": 0.10175459086894989, "sampling/sampling_logp_difference/max": 2.285191297531128, "sampling/sampling_logp_difference/mean": 0.01763945072889328, "step": 102, "step_time": 23.404857537010685 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1378.0, "completions/max_terminated_length": 1378.0, "completions/mean_length": 778.5799560546875, "completions/mean_terminated_length": 778.5799560546875, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "entropy": 0.20786661207675933, "epoch": 0.2798913043478261, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 6905677.0, "reward": 0.4198339879512787, "reward_std": 0.4877699613571167, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.38016602396965027, "rewards/length_penalty/std": 0.10589394718408585, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9925695657730103, "sampling/importance_sampling_ratio/min": 0.1209191381931305, "sampling/sampling_logp_difference/max": 2.112633228302002, "sampling/sampling_logp_difference/mean": 0.018408894538879395, "step": 103, "step_time": 16.658789492910728 }, { "clip_ratio/high_max": 0.001939025311730802, "clip_ratio/high_mean": 0.001939025311730802, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001939025311730802, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 1964.0, "completions/mean_length": 1361.280029296875, "completions/mean_terminated_length": 1094.2222900390625, "completions/min_length": 655.0, "completions/min_terminated_length": 655.0, "entropy": 0.2402542471885681, "epoch": 0.2826086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.024837521836161613, "learning_rate": 5e-05, "loss": 0.056361760944128036, "num_tokens": 6977251.0, "reward": -0.3046875, "reward_std": 0.6500034928321838, "rewards/correctness/mean": 0.36000001430511475, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.6646875143051147, "rewards/length_penalty/std": 0.2689364552497864, "sampling/importance_sampling_ratio/max": 2.514981269836426, "sampling/importance_sampling_ratio/mean": 0.991100549697876, "sampling/importance_sampling_ratio/min": 0.1535467803478241, "sampling/sampling_logp_difference/max": 1.8737499713897705, "sampling/sampling_logp_difference/mean": 0.01889841817319393, "step": 104, "step_time": 25.848534590099007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1821.0, "completions/max_terminated_length": 1821.0, "completions/mean_length": 1043.3599853515625, "completions/mean_terminated_length": 1043.3599853515625, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "entropy": 0.15879130959510804, "epoch": 0.28532608695652173, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 7031779.0, "reward": 0.29054686427116394, "reward_std": 0.5120186805725098, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5094531178474426, "rewards/length_penalty/std": 0.1817735731601715, "sampling/importance_sampling_ratio/max": 2.6664798259735107, "sampling/importance_sampling_ratio/mean": 0.9941168427467346, "sampling/importance_sampling_ratio/min": 0.16753947734832764, "sampling/sampling_logp_difference/max": 1.7865362167358398, "sampling/sampling_logp_difference/mean": 0.014325151219964027, "step": 105, "step_time": 21.45955632114783 }, { "clip_ratio/high_max": 0.0008601667475886642, "clip_ratio/high_mean": 0.0008601667475886642, "clip_ratio/low_mean": 4.555718332994729e-05, "clip_ratio/low_min": 4.555718332994729e-05, "clip_ratio/region_mean": 0.0009057239396497608, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1957.0, "completions/mean_length": 901.0999755859375, "completions/mean_terminated_length": 877.69384765625, "completions/min_length": 472.0, "completions/min_terminated_length": 472.0, "entropy": 0.18204533755779267, "epoch": 0.28804347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.02396683394908905, "learning_rate": 5e-05, "loss": 0.04690425843000412, "num_tokens": 7079314.0, "reward": 0.5200097560882568, "reward_std": 0.33328568935394287, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.43999022245407104, "rewards/length_penalty/std": 0.20017914474010468, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9932548999786377, "sampling/importance_sampling_ratio/min": 0.13250519335269928, "sampling/sampling_logp_difference/max": 2.0211334228515625, "sampling/sampling_logp_difference/mean": 0.016929278150200844, "step": 106, "step_time": 23.210859958082438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 1172.97998046875, "completions/mean_terminated_length": 954.2250366210938, "completions/min_length": 579.0, "completions/min_terminated_length": 579.0, "entropy": 0.21198867559432982, "epoch": 0.2907608695652174, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 7142003.0, "reward": 0.027255859225988388, "reward_std": 0.694027304649353, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.5727441310882568, "rewards/length_penalty/std": 0.23745079338550568, "sampling/importance_sampling_ratio/max": 1.8127131462097168, "sampling/importance_sampling_ratio/mean": 0.992159366607666, "sampling/importance_sampling_ratio/min": 0.209135964512825, "sampling/sampling_logp_difference/max": 1.5647706985473633, "sampling/sampling_logp_difference/mean": 0.016985761001706123, "step": 107, "step_time": 24.8778127937112 }, { "clip_ratio/high_max": 0.00020274321577744557, "clip_ratio/high_mean": 0.00020274321577744557, "clip_ratio/low_mean": 0.00019399074517423287, "clip_ratio/low_min": 0.00019399074517423287, "clip_ratio/region_mean": 0.0003967339551309124, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 1353.0, "completions/mean_length": 1105.4599609375, "completions/mean_terminated_length": 1023.5, "completions/min_length": 637.0, "completions/min_terminated_length": 637.0, "entropy": 0.19320686161518097, "epoch": 0.29347826086956524, "frac_reward_zero_std": 0.0, "grad_norm": 0.014451121911406517, "learning_rate": 5e-05, "loss": -0.0003622504882514477, "num_tokens": 7200326.0, "reward": 0.04022460803389549, "reward_std": 0.5708714723587036, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.5397753715515137, "rewards/length_penalty/std": 0.17200008034706116, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930944442749023, "sampling/importance_sampling_ratio/min": 0.11833740770816803, "sampling/sampling_logp_difference/max": 2.1342153549194336, "sampling/sampling_logp_difference/mean": 0.015908258035779, "step": 108, "step_time": 24.718877678969875 }, { "clip_ratio/high_max": 0.00020688481017714367, "clip_ratio/high_mean": 0.00020688481017714367, "clip_ratio/low_mean": 0.0001236838274053298, "clip_ratio/low_min": 0.0001236838274053298, "clip_ratio/region_mean": 0.00033056864631362257, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1905.0, "completions/mean_length": 1076.3599853515625, "completions/mean_terminated_length": 833.4500122070312, "completions/min_length": 420.0, "completions/min_terminated_length": 420.0, "entropy": 0.22805491089820862, "epoch": 0.296195652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.019839372485876083, "learning_rate": 5e-05, "loss": 0.01782216876745224, "num_tokens": 7256484.0, "reward": 0.21443359553813934, "reward_std": 0.6860597729682922, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.5255663990974426, "rewards/length_penalty/std": 0.27237775921821594, "sampling/importance_sampling_ratio/max": 1.8981941938400269, "sampling/importance_sampling_ratio/mean": 0.9908940196037292, "sampling/importance_sampling_ratio/min": 0.2387784719467163, "sampling/sampling_logp_difference/max": 1.4322190284729004, "sampling/sampling_logp_difference/mean": 0.01968378573656082, "step": 109, "step_time": 23.947757143061608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1528.0, "completions/max_terminated_length": 1528.0, "completions/mean_length": 812.8999633789062, "completions/mean_terminated_length": 812.8999633789062, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.17912407219409943, "epoch": 0.29891304347826086, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 7299709.0, "reward": 0.603076159954071, "reward_std": 0.1341657191514969, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.39692384004592896, "rewards/length_penalty/std": 0.1341657191514969, "sampling/importance_sampling_ratio/max": 2.312317132949829, "sampling/importance_sampling_ratio/mean": 0.9932844638824463, "sampling/importance_sampling_ratio/min": 0.19733035564422607, "sampling/sampling_logp_difference/max": 1.6228760480880737, "sampling/sampling_logp_difference/mean": 0.016147615388035774, "step": 110, "step_time": 17.61571263265796 }, { "clip_ratio/high_max": 0.001333821762818843, "clip_ratio/high_mean": 0.001333821762818843, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001333821762818843, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1951.0, "completions/mean_length": 895.8800048828125, "completions/mean_terminated_length": 872.3673095703125, "completions/min_length": 422.0, "completions/min_terminated_length": 422.0, "entropy": 0.1650071918964386, "epoch": 0.3016304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.023606708273291588, "learning_rate": 5e-05, "loss": 0.11822957545518875, "num_tokens": 7347163.0, "reward": 0.5225585699081421, "reward_std": 0.3198528587818146, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.4374414086341858, "rewards/length_penalty/std": 0.13831928372383118, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.993674635887146, "sampling/importance_sampling_ratio/min": 0.2151925414800644, "sampling/sampling_logp_difference/max": 1.746765375137329, "sampling/sampling_logp_difference/mean": 0.015003644861280918, "step": 111, "step_time": 23.701828924240544 }, { "clip_ratio/high_max": 0.00017219711444340646, "clip_ratio/high_mean": 0.00017219711444340646, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00017219711444340646, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1939.0, "completions/mean_length": 880.47998046875, "completions/mean_terminated_length": 831.8333740234375, "completions/min_length": 420.0, "completions/min_terminated_length": 420.0, "entropy": 0.21456990540027618, "epoch": 0.30434782608695654, "frac_reward_zero_std": 0.0, "grad_norm": 0.02423466555774212, "learning_rate": 5e-05, "loss": 0.01116170920431614, "num_tokens": 7393487.0, "reward": 0.07007812708616257, "reward_std": 0.5250024199485779, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.42992186546325684, "rewards/length_penalty/std": 0.20153114199638367, "sampling/importance_sampling_ratio/max": 2.3898568153381348, "sampling/importance_sampling_ratio/mean": 0.9927669167518616, "sampling/importance_sampling_ratio/min": 0.1668626368045807, "sampling/sampling_logp_difference/max": 1.7905843257904053, "sampling/sampling_logp_difference/mean": 0.017988070845603943, "step": 112, "step_time": 23.266813087509945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1250.0, "completions/max_terminated_length": 1250.0, "completions/mean_length": 717.4400024414062, "completions/mean_terminated_length": 717.4400024414062, "completions/min_length": 418.0, "completions/min_terminated_length": 418.0, "entropy": 0.1885949581861496, "epoch": 0.3070652173913043, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 7432669.0, "reward": 0.44968748092651367, "reward_std": 0.4758014380931854, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.3503125011920929, "rewards/length_penalty/std": 0.09486941993236542, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9926023483276367, "sampling/importance_sampling_ratio/min": 0.16496090590953827, "sampling/sampling_logp_difference/max": 1.802046775817871, "sampling/sampling_logp_difference/mean": 0.017040537670254707, "step": 113, "step_time": 14.779937229119241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1372.0, "completions/max_terminated_length": 1372.0, "completions/mean_length": 815.5199584960938, "completions/mean_terminated_length": 815.5199584960938, "completions/min_length": 537.0, "completions/min_terminated_length": 537.0, "entropy": 0.17523215413093568, "epoch": 0.30978260869565216, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 7475985.0, "reward": 0.6017968654632568, "reward_std": 0.12978872656822205, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.39820313453674316, "rewards/length_penalty/std": 0.12978872656822205, "sampling/importance_sampling_ratio/max": 2.5932438373565674, "sampling/importance_sampling_ratio/mean": 0.9938762187957764, "sampling/importance_sampling_ratio/min": 0.1701682060956955, "sampling/sampling_logp_difference/max": 1.7709678411483765, "sampling/sampling_logp_difference/mean": 0.016460731625556946, "step": 114, "step_time": 16.748307426227257 }, { "clip_ratio/high_max": 4.397053853608668e-05, "clip_ratio/high_mean": 4.397053853608668e-05, "clip_ratio/low_mean": 0.0001720914850011468, "clip_ratio/low_min": 0.0001720914850011468, "clip_ratio/region_mean": 0.00021606202353723346, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1901.0, "completions/mean_length": 958.719970703125, "completions/mean_terminated_length": 810.1818237304688, "completions/min_length": 457.0, "completions/min_terminated_length": 457.0, "entropy": 0.17697727978229522, "epoch": 0.3125, "frac_reward_zero_std": 0.0, "grad_norm": 0.021350838243961334, "learning_rate": 5e-05, "loss": 0.017744794487953186, "num_tokens": 7526151.0, "reward": 0.37187498807907104, "reward_std": 0.6004695892333984, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.46812498569488525, "rewards/length_penalty/std": 0.2550949454307556, "sampling/importance_sampling_ratio/max": 2.501585006713867, "sampling/importance_sampling_ratio/mean": 0.9931421279907227, "sampling/importance_sampling_ratio/min": 0.19160982966423035, "sampling/sampling_logp_difference/max": 1.6522941589355469, "sampling/sampling_logp_difference/mean": 0.01557265967130661, "step": 115, "step_time": 23.581499584252015 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1477.0, "completions/max_terminated_length": 1477.0, "completions/mean_length": 704.5199584960938, "completions/mean_terminated_length": 704.5199584960938, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "entropy": 0.17582472264766694, "epoch": 0.31521739130434784, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 7564267.0, "reward": 0.6559960842132568, "reward_std": 0.09484320878982544, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.34400391578674316, "rewards/length_penalty/std": 0.09484320878982544, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9932729005813599, "sampling/importance_sampling_ratio/min": 0.1512213796377182, "sampling/sampling_logp_difference/max": 1.9643642902374268, "sampling/sampling_logp_difference/mean": 0.016220567747950554, "step": 116, "step_time": 16.60399112198502 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 939.0, "completions/max_terminated_length": 939.0, "completions/mean_length": 604.5599975585938, "completions/mean_terminated_length": 604.5599975585938, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.17513634264469147, "epoch": 0.3179347826086957, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 7596675.0, "reward": 0.5048046708106995, "reward_std": 0.4149363338947296, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.2951953113079071, "rewards/length_penalty/std": 0.07410039752721786, "sampling/importance_sampling_ratio/max": 1.8821618556976318, "sampling/importance_sampling_ratio/mean": 0.9935084581375122, "sampling/importance_sampling_ratio/min": 0.2038552314043045, "sampling/sampling_logp_difference/max": 1.5903451442718506, "sampling/sampling_logp_difference/mean": 0.016780292615294456, "step": 117, "step_time": 11.17334417370148 }, { "clip_ratio/high_max": 0.0018218057695776225, "clip_ratio/high_mean": 0.0018218057695776225, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018218057695776225, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1501.0, "completions/mean_length": 788.3399658203125, "completions/mean_terminated_length": 762.6326293945312, "completions/min_length": 425.0, "completions/min_terminated_length": 425.0, "entropy": 0.22573850750923158, "epoch": 0.32065217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.02216523513197899, "learning_rate": 5e-05, "loss": 0.05535418540239334, "num_tokens": 7639772.0, "reward": 0.375068336725235, "reward_std": 0.4972207844257355, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.38493165373802185, "rewards/length_penalty/std": 0.14404763281345367, "sampling/importance_sampling_ratio/max": 2.038102865219116, "sampling/importance_sampling_ratio/mean": 0.9917938709259033, "sampling/importance_sampling_ratio/min": 0.17193739116191864, "sampling/sampling_logp_difference/max": 1.760624885559082, "sampling/sampling_logp_difference/mean": 0.019169943407177925, "step": 118, "step_time": 23.543540710117668 }, { "clip_ratio/high_max": 0.00028447814111132175, "clip_ratio/high_mean": 0.00028447814111132175, "clip_ratio/low_mean": 0.00020390173740452154, "clip_ratio/low_min": 0.00020390173740452154, "clip_ratio/region_mean": 0.0004883798785158433, "completions/clipped_ratio": 0.0, "completions/max_length": 1847.0, "completions/max_terminated_length": 1847.0, "completions/mean_length": 1012.7799682617188, "completions/mean_terminated_length": 1012.7799682617188, "completions/min_length": 479.0, "completions/min_terminated_length": 479.0, "entropy": 0.1529905915260315, "epoch": 0.3233695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.01852562092244625, "learning_rate": 5e-05, "loss": -0.016689183190464973, "num_tokens": 7693921.0, "reward": 0.28547850251197815, "reward_std": 0.5451766848564148, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.49452149868011475, "rewards/length_penalty/std": 0.176203653216362, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9942375421524048, "sampling/importance_sampling_ratio/min": 0.1694703847169876, "sampling/sampling_logp_difference/max": 1.7750771045684814, "sampling/sampling_logp_difference/mean": 0.0136912502348423, "step": 119, "step_time": 21.911348117981106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1120.0, "completions/max_terminated_length": 1120.0, "completions/mean_length": 674.0599975585938, "completions/mean_terminated_length": 674.0599975585938, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.17553678452968596, "epoch": 0.32608695652173914, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 7730474.0, "reward": 0.6708691120147705, "reward_std": 0.10527963191270828, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.3291308581829071, "rewards/length_penalty/std": 0.10527963936328888, "sampling/importance_sampling_ratio/max": 2.179398536682129, "sampling/importance_sampling_ratio/mean": 0.9938027262687683, "sampling/importance_sampling_ratio/min": 0.32254111766815186, "sampling/sampling_logp_difference/max": 1.131524682044983, "sampling/sampling_logp_difference/mean": 0.01581629365682602, "step": 120, "step_time": 13.308830082183704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1076.0, "completions/mean_length": 1000.2599487304688, "completions/mean_terminated_length": 738.3250122070312, "completions/min_length": 460.0, "completions/min_terminated_length": 460.0, "entropy": 0.23002321124076844, "epoch": 0.328804347826087, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 7783827.0, "reward": 0.11159179359674454, "reward_std": 0.6888429522514343, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.4884082078933716, "rewards/length_penalty/std": 0.2666991949081421, "sampling/importance_sampling_ratio/max": 2.023293972015381, "sampling/importance_sampling_ratio/mean": 0.9919323921203613, "sampling/importance_sampling_ratio/min": 0.279918909072876, "sampling/sampling_logp_difference/max": 1.2732553482055664, "sampling/sampling_logp_difference/mean": 0.017714740708470345, "step": 121, "step_time": 24.400762835517526 }, { "clip_ratio/high_max": 0.0005213107622694224, "clip_ratio/high_mean": 0.0005213107622694224, "clip_ratio/low_mean": 0.00022543201775988565, "clip_ratio/low_min": 0.00022543201775988565, "clip_ratio/region_mean": 0.0007467427873052657, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 1662.0, "completions/mean_length": 1128.7999267578125, "completions/mean_terminated_length": 869.5384521484375, "completions/min_length": 548.0, "completions/min_terminated_length": 548.0, "entropy": 0.2508073538541794, "epoch": 0.33152173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.02613849565386772, "learning_rate": 5e-05, "loss": 0.004862211644649506, "num_tokens": 7843387.0, "reward": 0.1688281148672104, "reward_std": 0.6833085417747498, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.5511718988418579, "rewards/length_penalty/std": 0.27070602774620056, "sampling/importance_sampling_ratio/max": 2.6110522747039795, "sampling/importance_sampling_ratio/mean": 0.9907427430152893, "sampling/importance_sampling_ratio/min": 0.20219014585018158, "sampling/sampling_logp_difference/max": 1.5985466241836548, "sampling/sampling_logp_difference/mean": 0.020016666501760483, "step": 122, "step_time": 24.24440438230522 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1685.0, "completions/max_terminated_length": 1685.0, "completions/mean_length": 861.8200073242188, "completions/mean_terminated_length": 861.8200073242188, "completions/min_length": 446.0, "completions/min_terminated_length": 446.0, "entropy": 0.18435944020748138, "epoch": 0.3342391304347826, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 7889718.0, "reward": 0.3791894316673279, "reward_std": 0.4217327833175659, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.4208105504512787, "rewards/length_penalty/std": 0.16375043988227844, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9926947355270386, "sampling/importance_sampling_ratio/min": 0.17068913578987122, "sampling/sampling_logp_difference/max": 1.7679113149642944, "sampling/sampling_logp_difference/mean": 0.01565755531191826, "step": 123, "step_time": 19.421543957898393 }, { "clip_ratio/high_max": 0.00019875064317602663, "clip_ratio/high_mean": 0.00019875064317602663, "clip_ratio/low_mean": 0.00019011201802641154, "clip_ratio/low_min": 0.00019011201802641154, "clip_ratio/region_mean": 0.0003888626612024382, "completions/clipped_ratio": 0.0, "completions/max_length": 1398.0, "completions/max_terminated_length": 1398.0, "completions/mean_length": 762.6599731445312, "completions/mean_terminated_length": 762.6599731445312, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.16533617079257965, "epoch": 0.33695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.013685276731848717, "learning_rate": 5e-05, "loss": -0.0037870663218200207, "num_tokens": 7930101.0, "reward": 0.40760740637779236, "reward_std": 0.44728195667266846, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.37239256501197815, "rewards/length_penalty/std": 0.15894412994384766, "sampling/importance_sampling_ratio/max": 2.962723970413208, "sampling/importance_sampling_ratio/mean": 0.9942588806152344, "sampling/importance_sampling_ratio/min": 0.27775922417640686, "sampling/sampling_logp_difference/max": 1.2810006141662598, "sampling/sampling_logp_difference/mean": 0.014873773790895939, "step": 124, "step_time": 16.265020058257505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1140.0, "completions/max_terminated_length": 1140.0, "completions/mean_length": 612.8800048828125, "completions/mean_terminated_length": 612.8800048828125, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.19633746445178984, "epoch": 0.33967391304347827, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 7963225.0, "reward": 0.5007421970367432, "reward_std": 0.439824640750885, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.2992578148841858, "rewards/length_penalty/std": 0.0784917026758194, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9923263788223267, "sampling/importance_sampling_ratio/min": 0.2048402577638626, "sampling/sampling_logp_difference/max": 1.5855247974395752, "sampling/sampling_logp_difference/mean": 0.018205469474196434, "step": 125, "step_time": 13.66795008117333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 1744.0, "completions/mean_length": 1158.5599365234375, "completions/mean_terminated_length": 740.0, "completions/min_length": 419.0, "completions/min_terminated_length": 419.0, "entropy": 0.23917241990566254, "epoch": 0.3423913043478261, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 8026123.0, "reward": -0.16570311784744263, "reward_std": 0.7524628043174744, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.5657031536102295, "rewards/length_penalty/std": 0.3224644064903259, "sampling/importance_sampling_ratio/max": 1.7765871286392212, "sampling/importance_sampling_ratio/mean": 0.9911412596702576, "sampling/importance_sampling_ratio/min": 0.26575323939323425, "sampling/sampling_logp_difference/max": 1.325187087059021, "sampling/sampling_logp_difference/mean": 0.018271170556545258, "step": 126, "step_time": 24.908350980607793 }, { "clip_ratio/high_max": 0.00045200209133327006, "clip_ratio/high_mean": 0.00045200209133327006, "clip_ratio/low_mean": 0.0001414491154719144, "clip_ratio/low_min": 0.0001414491154719144, "clip_ratio/region_mean": 0.0005934512009844184, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1863.0, "completions/mean_length": 888.6799926757812, "completions/mean_terminated_length": 759.86669921875, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.2037830114364624, "epoch": 0.3451086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.019005078822374344, "learning_rate": 5e-05, "loss": 0.04123948514461517, "num_tokens": 8073187.0, "reward": 0.4460742175579071, "reward_std": 0.530727744102478, "rewards/correctness/mean": 0.8799999952316284, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.4339257776737213, "rewards/length_penalty/std": 0.2465292364358902, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9926847219467163, "sampling/importance_sampling_ratio/min": 0.25655215978622437, "sampling/sampling_logp_difference/max": 1.3604233264923096, "sampling/sampling_logp_difference/mean": 0.01663067378103733, "step": 127, "step_time": 23.373054043157026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1433.0, "completions/max_terminated_length": 1433.0, "completions/mean_length": 820.8800048828125, "completions/mean_terminated_length": 820.8800048828125, "completions/min_length": 486.0, "completions/min_terminated_length": 486.0, "entropy": 0.16171946823596955, "epoch": 0.34782608695652173, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 8116551.0, "reward": 0.5991796851158142, "reward_std": 0.1196410283446312, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.4008203148841858, "rewards/length_penalty/std": 0.1196410283446312, "sampling/importance_sampling_ratio/max": 2.0136053562164307, "sampling/importance_sampling_ratio/mean": 0.9934844970703125, "sampling/importance_sampling_ratio/min": 0.31500232219696045, "sampling/sampling_logp_difference/max": 1.1551752090454102, "sampling/sampling_logp_difference/mean": 0.014645514078438282, "step": 128, "step_time": 17.13064171303995 }, { "clip_ratio/high_max": 0.0002891793905291706, "clip_ratio/high_mean": 0.0002891793905291706, "clip_ratio/low_mean": 0.00016118580533657222, "clip_ratio/low_min": 0.00016118580533657222, "clip_ratio/region_mean": 0.0004503651871345937, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 947.6399536132812, "completions/mean_terminated_length": 925.1836547851562, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.14684489071369172, "epoch": 0.35054347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.017561012879014015, "learning_rate": 5e-05, "loss": 0.02268596738576889, "num_tokens": 8166993.0, "reward": 0.07728515565395355, "reward_std": 0.5825548768043518, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.4627148509025574, "rewards/length_penalty/std": 0.21437303721904755, "sampling/importance_sampling_ratio/max": 2.030771017074585, "sampling/importance_sampling_ratio/mean": 0.9945068359375, "sampling/importance_sampling_ratio/min": 0.191323921084404, "sampling/sampling_logp_difference/max": 1.65378737449646, "sampling/sampling_logp_difference/mean": 0.01251893863081932, "step": 129, "step_time": 23.582299377769232 }, { "clip_ratio/high_max": 0.00020219890866428613, "clip_ratio/high_mean": 0.00020219890866428613, "clip_ratio/low_mean": 7.859817269491031e-05, "clip_ratio/low_min": 7.859817269491031e-05, "clip_ratio/region_mean": 0.00028079708135919645, "completions/clipped_ratio": 0.17999999225139618, "completions/max_length": 2048.0, "completions/max_terminated_length": 1912.0, "completions/mean_length": 926.5999755859375, "completions/mean_terminated_length": 680.4390258789062, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.28622645139694214, "epoch": 0.3532608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.027635367587208748, "learning_rate": 5e-05, "loss": 0.008104096166789532, "num_tokens": 8216093.0, "reward": 0.3675585985183716, "reward_std": 0.6559378504753113, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.45244139432907104, "rewards/length_penalty/std": 0.2801753580570221, "sampling/importance_sampling_ratio/max": 2.239715814590454, "sampling/importance_sampling_ratio/mean": 0.988741934299469, "sampling/importance_sampling_ratio/min": 0.33027565479278564, "sampling/sampling_logp_difference/max": 1.1078276634216309, "sampling/sampling_logp_difference/mean": 0.021850842982530594, "step": 130, "step_time": 23.556175739970058 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1782.0, "completions/max_terminated_length": 1782.0, "completions/mean_length": 1049.4599609375, "completions/mean_terminated_length": 1049.4599609375, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "entropy": 0.17545301020145415, "epoch": 0.35597826086956524, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 8271946.0, "reward": 0.2875683605670929, "reward_std": 0.45490753650665283, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5124316215515137, "rewards/length_penalty/std": 0.16384990513324738, "sampling/importance_sampling_ratio/max": 1.8399344682693481, "sampling/importance_sampling_ratio/mean": 0.9932869076728821, "sampling/importance_sampling_ratio/min": 0.23934902250766754, "sampling/sampling_logp_difference/max": 1.4298324584960938, "sampling/sampling_logp_difference/mean": 0.014576495625078678, "step": 131, "step_time": 21.70218894118443 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1180.0, "completions/max_terminated_length": 1180.0, "completions/mean_length": 814.0799560546875, "completions/mean_terminated_length": 814.0799560546875, "completions/min_length": 501.0, "completions/min_terminated_length": 501.0, "entropy": 0.15156003534793855, "epoch": 0.358695652173913, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 8314480.0, "reward": 0.4025000035762787, "reward_std": 0.3871462941169739, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.39750000834465027, "rewards/length_penalty/std": 0.0840296596288681, "sampling/importance_sampling_ratio/max": 1.7450766563415527, "sampling/importance_sampling_ratio/mean": 0.9944218993186951, "sampling/importance_sampling_ratio/min": 0.15414585173130035, "sampling/sampling_logp_difference/max": 1.8698559999465942, "sampling/sampling_logp_difference/mean": 0.01384400948882103, "step": 132, "step_time": 14.282176236156374 }, { "clip_ratio/high_max": 0.00046791560016572474, "clip_ratio/high_mean": 0.00046791560016572474, "clip_ratio/low_mean": 8.201263844966888e-05, "clip_ratio/low_min": 8.201263844966888e-05, "clip_ratio/region_mean": 0.0005499282269738615, "completions/clipped_ratio": 0.0, "completions/max_length": 2033.0, "completions/max_terminated_length": 2033.0, "completions/mean_length": 925.4599609375, "completions/mean_terminated_length": 925.4599609375, "completions/min_length": 170.0, "completions/min_terminated_length": 170.0, "entropy": 0.16496414840221404, "epoch": 0.36141304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.026071475818753242, "learning_rate": 5e-05, "loss": 0.05878584086894989, "num_tokens": 8363443.0, "reward": 0.5081152319908142, "reward_std": 0.32649436593055725, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.45188477635383606, "rewards/length_penalty/std": 0.20533433556556702, "sampling/importance_sampling_ratio/max": 2.9383115768432617, "sampling/importance_sampling_ratio/mean": 0.9940788149833679, "sampling/importance_sampling_ratio/min": 0.22200119495391846, "sampling/sampling_logp_difference/max": 1.5050725936889648, "sampling/sampling_logp_difference/mean": 0.013843018561601639, "step": 133, "step_time": 23.062475404934958 }, { "clip_ratio/high_max": 0.0005703021073713898, "clip_ratio/high_mean": 0.0005703021073713898, "clip_ratio/low_mean": 0.00012266641133464872, "clip_ratio/low_min": 0.00012266641133464872, "clip_ratio/region_mean": 0.0006929685128852725, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 2000.0, "completions/mean_length": 780.3800048828125, "completions/mean_terminated_length": 727.5625, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "entropy": 0.2758919417858124, "epoch": 0.3641304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.02310226857662201, "learning_rate": 5e-05, "loss": 0.02851085551083088, "num_tokens": 8405152.0, "reward": 0.4589550793170929, "reward_std": 0.5565141439437866, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.3810449242591858, "rewards/length_penalty/std": 0.22435730695724487, "sampling/importance_sampling_ratio/max": 2.6123244762420654, "sampling/importance_sampling_ratio/mean": 0.9894843101501465, "sampling/importance_sampling_ratio/min": 0.3154922127723694, "sampling/sampling_logp_difference/max": 1.1536211967468262, "sampling/sampling_logp_difference/mean": 0.02216336503624916, "step": 134, "step_time": 22.863046364858747 }, { "clip_ratio/high_max": 0.0007523948675952852, "clip_ratio/high_mean": 0.0007523948675952852, "clip_ratio/low_mean": 0.00011037255899282172, "clip_ratio/low_min": 0.00011037255899282172, "clip_ratio/region_mean": 0.0008627674425952137, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 2044.0, "completions/mean_length": 1299.659912109375, "completions/mean_terminated_length": 1036.729736328125, "completions/min_length": 709.0, "completions/min_terminated_length": 709.0, "entropy": 0.2696214348077774, "epoch": 0.36684782608695654, "frac_reward_zero_std": 0.0, "grad_norm": 0.022844277322292328, "learning_rate": 5e-05, "loss": 0.005371954292058945, "num_tokens": 8474315.0, "reward": -0.014599609188735485, "reward_std": 0.7156080603599548, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.6345996260643005, "rewards/length_penalty/std": 0.2660164535045624, "sampling/importance_sampling_ratio/max": 1.8941152095794678, "sampling/importance_sampling_ratio/mean": 0.9900227189064026, "sampling/importance_sampling_ratio/min": 0.3891741931438446, "sampling/sampling_logp_difference/max": 0.9437282085418701, "sampling/sampling_logp_difference/mean": 0.01943516731262207, "step": 135, "step_time": 25.70930730109103 }, { "clip_ratio/high_max": 0.0007099440554156899, "clip_ratio/high_mean": 0.0007099440554156899, "clip_ratio/low_mean": 0.00016588732833042741, "clip_ratio/low_min": 0.00016588732833042741, "clip_ratio/region_mean": 0.0008758313604630529, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1936.0, "completions/mean_length": 961.8200073242188, "completions/mean_terminated_length": 939.6530151367188, "completions/min_length": 612.0, "completions/min_terminated_length": 612.0, "entropy": 0.2529032438993454, "epoch": 0.3695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.02607668936252594, "learning_rate": 5e-05, "loss": 0.0517682209610939, "num_tokens": 8524486.0, "reward": 0.29036131501197815, "reward_std": 0.4229385554790497, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.4696386754512787, "rewards/length_penalty/std": 0.14740881323814392, "sampling/importance_sampling_ratio/max": 1.8573329448699951, "sampling/importance_sampling_ratio/mean": 0.9908045530319214, "sampling/importance_sampling_ratio/min": 0.2634143531322479, "sampling/sampling_logp_difference/max": 1.3340270519256592, "sampling/sampling_logp_difference/mean": 0.019505880773067474, "step": 136, "step_time": 23.253381483722478 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1496.0, "completions/max_terminated_length": 1496.0, "completions/mean_length": 799.239990234375, "completions/mean_terminated_length": 799.239990234375, "completions/min_length": 451.0, "completions/min_terminated_length": 451.0, "entropy": 0.1512964367866516, "epoch": 0.37228260869565216, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 8567308.0, "reward": 0.009746093302965164, "reward_std": 0.5480278730392456, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.3902539014816284, "rewards/length_penalty/std": 0.11690854281187057, "sampling/importance_sampling_ratio/max": 1.6863409280776978, "sampling/importance_sampling_ratio/mean": 0.994084894657135, "sampling/importance_sampling_ratio/min": 0.3348821699619293, "sampling/sampling_logp_difference/max": 1.0939764976501465, "sampling/sampling_logp_difference/mean": 0.01341221947222948, "step": 137, "step_time": 17.29949672264047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1785.0, "completions/max_terminated_length": 1785.0, "completions/mean_length": 744.4199829101562, "completions/mean_terminated_length": 744.4199829101562, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "entropy": 0.18776478469371796, "epoch": 0.375, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 8607559.0, "reward": 0.2365136593580246, "reward_std": 0.6176913380622864, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.36348631978034973, "rewards/length_penalty/std": 0.16838611662387848, "sampling/importance_sampling_ratio/max": 2.3696627616882324, "sampling/importance_sampling_ratio/mean": 0.9930192828178406, "sampling/importance_sampling_ratio/min": 0.29692041873931885, "sampling/sampling_logp_difference/max": 1.2142912149429321, "sampling/sampling_logp_difference/mean": 0.015935169532895088, "step": 138, "step_time": 20.486768319038674 }, { "clip_ratio/high_max": 0.0007103165335138328, "clip_ratio/high_mean": 0.0007103165335138328, "clip_ratio/low_mean": 9.246863919543102e-05, "clip_ratio/low_min": 9.246863919543102e-05, "clip_ratio/region_mean": 0.000802785181440413, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 2012.0, "completions/mean_length": 939.1799926757812, "completions/mean_terminated_length": 842.7608642578125, "completions/min_length": 425.0, "completions/min_terminated_length": 425.0, "entropy": 0.23389012813568116, "epoch": 0.37771739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.022810379043221474, "learning_rate": 5e-05, "loss": 0.00583074614405632, "num_tokens": 8657298.0, "reward": 0.22141601145267487, "reward_std": 0.5956475138664246, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.4585839807987213, "rewards/length_penalty/std": 0.256958544254303, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9912944436073303, "sampling/importance_sampling_ratio/min": 0.18154411017894745, "sampling/sampling_logp_difference/max": 1.706256628036499, "sampling/sampling_logp_difference/mean": 0.01880025304853916, "step": 139, "step_time": 23.663719915319234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.17999999225139618, "completions/max_length": 2048.0, "completions/max_terminated_length": 1984.0, "completions/mean_length": 950.8999633789062, "completions/mean_terminated_length": 710.0731201171875, "completions/min_length": 481.0, "completions/min_terminated_length": 481.0, "entropy": 0.2804166585206985, "epoch": 0.3804347826086957, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 8708483.0, "reward": 0.335693359375, "reward_std": 0.6745652556419373, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.46430665254592896, "rewards/length_penalty/std": 0.27276018261909485, "sampling/importance_sampling_ratio/max": 2.0790176391601562, "sampling/importance_sampling_ratio/mean": 0.9897128343582153, "sampling/importance_sampling_ratio/min": 0.18834657967090607, "sampling/sampling_logp_difference/max": 1.6694715023040771, "sampling/sampling_logp_difference/mean": 0.02086036093533039, "step": 140, "step_time": 23.888739782385528 }, { "clip_ratio/high_max": 0.0001346932433079928, "clip_ratio/high_mean": 0.0001346932433079928, "clip_ratio/low_mean": 7.464078662451356e-05, "clip_ratio/low_min": 7.464078662451356e-05, "clip_ratio/region_mean": 0.00020933403575327248, "completions/clipped_ratio": 0.0, "completions/max_length": 1432.0, "completions/max_terminated_length": 1432.0, "completions/mean_length": 787.8399658203125, "completions/mean_terminated_length": 787.8399658203125, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.1694457083940506, "epoch": 0.38315217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.019208740442991257, "learning_rate": 5e-05, "loss": 0.004709660075604916, "num_tokens": 8750605.0, "reward": 0.39531248807907104, "reward_std": 0.40742599964141846, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.38468751311302185, "rewards/length_penalty/std": 0.15147896111011505, "sampling/importance_sampling_ratio/max": 1.9426826238632202, "sampling/importance_sampling_ratio/mean": 0.9936075210571289, "sampling/importance_sampling_ratio/min": 0.3054625689983368, "sampling/sampling_logp_difference/max": 1.1859281063079834, "sampling/sampling_logp_difference/mean": 0.014435612596571445, "step": 141, "step_time": 16.739115350646898 }, { "clip_ratio/high_max": 0.0009548861533403396, "clip_ratio/high_mean": 0.0009548861533403396, "clip_ratio/low_mean": 1.7024175031110643e-05, "clip_ratio/low_min": 1.7024175031110643e-05, "clip_ratio/region_mean": 0.0009719103341922164, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1947.0, "completions/mean_length": 1021.2999877929688, "completions/mean_terminated_length": 881.2954711914062, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 0.19140491485595704, "epoch": 0.3858695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.023622607812285423, "learning_rate": 5e-05, "loss": 0.023460185155272484, "num_tokens": 8804460.0, "reward": 0.4213183522224426, "reward_std": 0.4741456210613251, "rewards/correctness/mean": 0.9200000166893005, "rewards/correctness/std": 0.27404749393463135, "rewards/length_penalty/mean": -0.4986816346645355, "rewards/length_penalty/std": 0.26050740480422974, "sampling/importance_sampling_ratio/max": 2.3221938610076904, "sampling/importance_sampling_ratio/mean": 0.9930033087730408, "sampling/importance_sampling_ratio/min": 0.2887270748615265, "sampling/sampling_logp_difference/max": 1.2422734498977661, "sampling/sampling_logp_difference/mean": 0.015386838465929031, "step": 142, "step_time": 24.441725069656968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1273.0, "completions/max_terminated_length": 1273.0, "completions/mean_length": 816.8999633789062, "completions/mean_terminated_length": 816.8999633789062, "completions/min_length": 459.0, "completions/min_terminated_length": 459.0, "entropy": 0.1633100241422653, "epoch": 0.38858695652173914, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 8847645.0, "reward": 0.401123046875, "reward_std": 0.43309485912323, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.39887696504592896, "rewards/length_penalty/std": 0.09776387363672256, "sampling/importance_sampling_ratio/max": 1.852404236793518, "sampling/importance_sampling_ratio/mean": 0.9938700199127197, "sampling/importance_sampling_ratio/min": 0.3081684410572052, "sampling/sampling_logp_difference/max": 1.1771087646484375, "sampling/sampling_logp_difference/mean": 0.014416918158531189, "step": 143, "step_time": 15.25021388544701 }, { "clip_ratio/high_max": 0.00016280016861855984, "clip_ratio/high_mean": 0.00016280016861855984, "clip_ratio/low_mean": 0.00013785602495772765, "clip_ratio/low_min": 0.00013785602495772765, "clip_ratio/region_mean": 0.0003006561935762875, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 2035.0, "completions/mean_length": 1043.9599609375, "completions/mean_terminated_length": 932.4000244140625, "completions/min_length": 451.0, "completions/min_terminated_length": 451.0, "entropy": 0.22924869060516356, "epoch": 0.391304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.02698708325624466, "learning_rate": 5e-05, "loss": -0.0073261559009552, "num_tokens": 8902893.0, "reward": 0.1302538961172104, "reward_std": 0.5761489272117615, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.5097460746765137, "rewards/length_penalty/std": 0.2696961462497711, "sampling/importance_sampling_ratio/max": 2.141483783721924, "sampling/importance_sampling_ratio/mean": 0.9910635948181152, "sampling/importance_sampling_ratio/min": 0.224883571267128, "sampling/sampling_logp_difference/max": 1.4921724796295166, "sampling/sampling_logp_difference/mean": 0.018297677859663963, "step": 144, "step_time": 24.027998832287267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1853.0, "completions/max_terminated_length": 1853.0, "completions/mean_length": 719.1399536132812, "completions/mean_terminated_length": 719.1399536132812, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.250396528840065, "epoch": 0.39402173913043476, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 8941180.0, "reward": 0.24885742366313934, "reward_std": 0.5498008728027344, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.3511425852775574, "rewards/length_penalty/std": 0.15510118007659912, "sampling/importance_sampling_ratio/max": 2.0451557636260986, "sampling/importance_sampling_ratio/mean": 0.991004467010498, "sampling/importance_sampling_ratio/min": 0.3119693696498871, "sampling/sampling_logp_difference/max": 1.1648502349853516, "sampling/sampling_logp_difference/mean": 0.019350690767169, "step": 145, "step_time": 20.871423684293404 }, { "clip_ratio/high_max": 0.00040537488530389965, "clip_ratio/high_mean": 0.00040537488530389965, "clip_ratio/low_mean": 0.00013418944436125458, "clip_ratio/low_min": 0.00013418944436125458, "clip_ratio/region_mean": 0.0005395643354859203, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1520.0, "completions/mean_length": 846.4599609375, "completions/mean_terminated_length": 821.9387817382812, "completions/min_length": 387.0, "completions/min_terminated_length": 387.0, "entropy": 0.17892795503139497, "epoch": 0.3967391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.02788802795112133, "learning_rate": 5e-05, "loss": 0.06696762144565582, "num_tokens": 8986443.0, "reward": 0.5666894316673279, "reward_std": 0.25304287672042847, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.41331055760383606, "rewards/length_penalty/std": 0.14171867072582245, "sampling/importance_sampling_ratio/max": 2.091669797897339, "sampling/importance_sampling_ratio/mean": 0.9935770630836487, "sampling/importance_sampling_ratio/min": 0.19649873673915863, "sampling/sampling_logp_difference/max": 1.6270992755889893, "sampling/sampling_logp_difference/mean": 0.014963977038860321, "step": 146, "step_time": 22.842710306169465 }, { "clip_ratio/high_max": 0.00020341041235951706, "clip_ratio/high_mean": 0.00020341041235951706, "clip_ratio/low_mean": 0.0002711974200792611, "clip_ratio/low_min": 0.0002711974200792611, "clip_ratio/region_mean": 0.0004746078397147357, "completions/clipped_ratio": 0.0, "completions/max_length": 1644.0, "completions/max_terminated_length": 1644.0, "completions/mean_length": 831.8399658203125, "completions/mean_terminated_length": 831.8399658203125, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.16965317726135254, "epoch": 0.39945652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.019360462203621864, "learning_rate": 5e-05, "loss": -0.0011868514120578766, "num_tokens": 9030335.0, "reward": 0.5738281011581421, "reward_std": 0.22484731674194336, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.40617188811302185, "rewards/length_penalty/std": 0.14953912794589996, "sampling/importance_sampling_ratio/max": 1.920053482055664, "sampling/importance_sampling_ratio/mean": 0.9935393929481506, "sampling/importance_sampling_ratio/min": 0.29376471042633057, "sampling/sampling_logp_difference/max": 1.2249760627746582, "sampling/sampling_logp_difference/mean": 0.01462612021714449, "step": 147, "step_time": 18.802571587264538 }, { "clip_ratio/high_max": 0.00014153838419588282, "clip_ratio/high_mean": 0.00014153838419588282, "clip_ratio/low_mean": 0.0002659023535670713, "clip_ratio/low_min": 0.0002659023535670713, "clip_ratio/region_mean": 0.00040744072757661345, "completions/clipped_ratio": 0.0, "completions/max_length": 1523.0, "completions/max_terminated_length": 1523.0, "completions/mean_length": 836.9199829101562, "completions/mean_terminated_length": 836.9199829101562, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.18687565624713898, "epoch": 0.40217391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.015342357568442822, "learning_rate": 5e-05, "loss": -0.005616676993668079, "num_tokens": 9075091.0, "reward": 0.5713476538658142, "reward_std": 0.2033325433731079, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.40865233540534973, "rewards/length_penalty/std": 0.16827945411205292, "sampling/importance_sampling_ratio/max": 1.9206771850585938, "sampling/importance_sampling_ratio/mean": 0.9928330779075623, "sampling/importance_sampling_ratio/min": 0.23788392543792725, "sampling/sampling_logp_difference/max": 1.4359724521636963, "sampling/sampling_logp_difference/mean": 0.015589497052133083, "step": 148, "step_time": 17.858083135215566 }, { "clip_ratio/high_max": 0.0005155283433850855, "clip_ratio/high_mean": 0.0005155283433850855, "clip_ratio/low_mean": 3.542958293110132e-05, "clip_ratio/low_min": 3.542958293110132e-05, "clip_ratio/region_mean": 0.0005509579263161868, "completions/clipped_ratio": 0.0, "completions/max_length": 1625.0, "completions/max_terminated_length": 1625.0, "completions/mean_length": 1159.7999267578125, "completions/mean_terminated_length": 1159.7999267578125, "completions/min_length": 601.0, "completions/min_terminated_length": 601.0, "entropy": 0.1700785607099533, "epoch": 0.4048913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.01995597593486309, "learning_rate": 5e-05, "loss": -0.009008022025227547, "num_tokens": 9136051.0, "reward": -0.04630859196186066, "reward_std": 0.5454481840133667, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.5663086175918579, "rewards/length_penalty/std": 0.11681854724884033, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9939435720443726, "sampling/importance_sampling_ratio/min": 0.18416427075862885, "sampling/sampling_logp_difference/max": 1.691927194595337, "sampling/sampling_logp_difference/mean": 0.013781562447547913, "step": 149, "step_time": 20.784359193407 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1185.0, "completions/max_terminated_length": 1185.0, "completions/mean_length": 649.8599853515625, "completions/mean_terminated_length": 649.8599853515625, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.17341043949127197, "epoch": 0.4076086956521739, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 9170304.0, "reward": 0.6826855540275574, "reward_std": 0.11302065849304199, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.3173144459724426, "rewards/length_penalty/std": 0.11302065849304199, "sampling/importance_sampling_ratio/max": 2.075118064880371, "sampling/importance_sampling_ratio/mean": 0.9931694865226746, "sampling/importance_sampling_ratio/min": 0.36798980832099915, "sampling/sampling_logp_difference/max": 0.999700129032135, "sampling/sampling_logp_difference/mean": 0.01535247266292572, "step": 150, "step_time": 13.658987378934398 } ], "logging_steps": 1, "max_steps": 200, "num_input_tokens_seen": 9170304, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 10, "trial_name": null, "trial_params": null }