{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.5830903790087464, "eval_steps": 500, "global_step": 400, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1779.0, "completions/mean_length": 911.7000732421875, "completions/mean_terminated_length": 851.8947143554688, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.287322332461675, "epoch": 0.0014577259475218659, "frac_reward_zero_std": 0.0, "grad_norm": 0.009315529838204384, "learning_rate": 0.0, "loss": -0.00722887646406889, "num_tokens": 59102.0, "reward": 0.02150065265595913, "reward_std": 0.6429958939552307, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.4451660215854645, "rewards/length_penalty/std": 0.22199024260044098, "sampling/importance_sampling_ratio/max": 1.4242706298828125, "sampling/importance_sampling_ratio/mean": 0.9905919432640076, "sampling/importance_sampling_ratio/min": 0.39864858984947205, "sampling/sampling_logp_difference/max": 0.9196749925613403, "sampling/sampling_logp_difference/mean": 0.017188014462590218, "step": 1, "step_time": 27.266703790985048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1218.0, "completions/mean_length": 583.9833374023438, "completions/mean_terminated_length": 559.1694946289062, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "entropy": 0.18992788344621658, "epoch": 0.0029154518950437317, "frac_reward_zero_std": 0.0, "grad_norm": 0.01012346614152193, "learning_rate": 3.3333333333333335e-07, "loss": 0.0682297945022583, "num_tokens": 97601.0, "reward": 0.39818525314331055, "reward_std": 0.5346248745918274, "rewards/correctness/mean": 0.6833333373069763, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.28514811396598816, "rewards/length_penalty/std": 0.16064155101776123, "sampling/importance_sampling_ratio/max": 1.4211019277572632, "sampling/importance_sampling_ratio/mean": 0.9934695363044739, "sampling/importance_sampling_ratio/min": 0.6183077096939087, "sampling/sampling_logp_difference/max": 0.4807690382003784, "sampling/sampling_logp_difference/mean": 0.011985952034592628, "step": 2, "step_time": 24.735876325285062 }, { "clip_ratio/high_max": 0.0005826703830583332, "clip_ratio/high_mean": 0.0005826703830583332, "clip_ratio/low_mean": 0.00020714851901478445, "clip_ratio/low_min": 0.00020714851901478445, "clip_ratio/region_mean": 0.000789818882670564, "completions/clipped_ratio": 0.18333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 1824.0, "completions/mean_length": 1067.4000244140625, "completions/mean_terminated_length": 847.2652587890625, "completions/min_length": 468.0, "completions/min_terminated_length": 468.0, "entropy": 0.3251006404558818, "epoch": 0.004373177842565598, "frac_reward_zero_std": 0.0, "grad_norm": 0.015216746367514133, "learning_rate": 6.666666666666667e-07, "loss": 0.07915706932544708, "num_tokens": 167245.0, "reward": -0.0545247420668602, "reward_std": 0.721366822719574, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.521191418170929, "rewards/length_penalty/std": 0.289736270904541, "sampling/importance_sampling_ratio/max": 1.4119378328323364, "sampling/importance_sampling_ratio/mean": 0.9888848662376404, "sampling/importance_sampling_ratio/min": 0.5514586567878723, "sampling/sampling_logp_difference/max": 0.5951883792877197, "sampling/sampling_logp_difference/mean": 0.018833020702004433, "step": 3, "step_time": 28.364217409398407 }, { "clip_ratio/high_max": 0.0009626642034466689, "clip_ratio/high_mean": 0.0009626642034466689, "clip_ratio/low_mean": 8.59517022036016e-05, "clip_ratio/low_min": 8.59517022036016e-05, "clip_ratio/region_mean": 0.0010486159202021856, "completions/clipped_ratio": 0.0, "completions/max_length": 1864.0, "completions/max_terminated_length": 1864.0, "completions/mean_length": 536.9000244140625, "completions/mean_terminated_length": 536.9000244140625, "completions/min_length": 59.0, "completions/min_terminated_length": 59.0, "entropy": 0.2079171563188235, "epoch": 0.0058309037900874635, "frac_reward_zero_std": 0.0, "grad_norm": 0.009288053959608078, "learning_rate": 1.0000000000000002e-06, "loss": 0.0008611474186182022, "num_tokens": 202649.0, "reward": 0.32117515802383423, "reward_std": 0.5547496676445007, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.26215821504592896, "rewards/length_penalty/std": 0.22235317528247833, "sampling/importance_sampling_ratio/max": 1.4492052793502808, "sampling/importance_sampling_ratio/mean": 0.9927003383636475, "sampling/importance_sampling_ratio/min": 0.673969566822052, "sampling/sampling_logp_difference/max": 0.39457035064697266, "sampling/sampling_logp_difference/mean": 0.013110565021634102, "step": 4, "step_time": 22.774585011880845 }, { "clip_ratio/high_max": 0.0007607487593001375, "clip_ratio/high_mean": 0.0007607487593001375, "clip_ratio/low_mean": 0.00016298730891624777, "clip_ratio/low_min": 0.00016298730891624777, "clip_ratio/region_mean": 0.000923736069429045, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1981.0, "completions/mean_length": 957.6000366210938, "completions/mean_terminated_length": 836.4444580078125, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.2149273157119751, "epoch": 0.007288629737609329, "frac_reward_zero_std": 0.0, "grad_norm": 0.012508715502917767, "learning_rate": 1.3333333333333334e-06, "loss": 0.05253123119473457, "num_tokens": 265495.0, "reward": 0.16575521230697632, "reward_std": 0.6683671474456787, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.46757811307907104, "rewards/length_penalty/std": 0.2820192873477936, "sampling/importance_sampling_ratio/max": 1.648134708404541, "sampling/importance_sampling_ratio/mean": 0.9923359155654907, "sampling/importance_sampling_ratio/min": 0.6768563985824585, "sampling/sampling_logp_difference/max": 0.4996441602706909, "sampling/sampling_logp_difference/mean": 0.013547893613576889, "step": 5, "step_time": 28.106884445995092 }, { "clip_ratio/high_max": 0.0007083940787803537, "clip_ratio/high_mean": 0.0007083940787803537, "clip_ratio/low_mean": 0.00020570932732274136, "clip_ratio/low_min": 0.00020570932732274136, "clip_ratio/region_mean": 0.0009141034194423506, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1958.0, "completions/mean_length": 969.86669921875, "completions/mean_terminated_length": 754.239990234375, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.2867318441470464, "epoch": 0.008746355685131196, "frac_reward_zero_std": 0.0, "grad_norm": 0.012962858192622662, "learning_rate": 1.6666666666666667e-06, "loss": 0.032503724098205566, "num_tokens": 328167.0, "reward": 0.12643229961395264, "reward_std": 0.7710843086242676, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.4735676944255829, "rewards/length_penalty/std": 0.31948766112327576, "sampling/importance_sampling_ratio/max": 1.4769279956817627, "sampling/importance_sampling_ratio/mean": 0.9899801015853882, "sampling/importance_sampling_ratio/min": 0.5725703835487366, "sampling/sampling_logp_difference/max": 0.557619571685791, "sampling/sampling_logp_difference/mean": 0.01695270836353302, "step": 6, "step_time": 27.56186681916006 }, { "clip_ratio/high_max": 0.0011398453304233651, "clip_ratio/high_mean": 0.0011398453304233651, "clip_ratio/low_mean": 6.085470522521064e-05, "clip_ratio/low_min": 6.085470522521064e-05, "clip_ratio/region_mean": 0.0012007000429245334, "completions/clipped_ratio": 0.0, "completions/max_length": 1098.0, "completions/max_terminated_length": 1098.0, "completions/mean_length": 542.3833618164062, "completions/mean_terminated_length": 542.3833618164062, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.17402570943037668, "epoch": 0.01020408163265306, "frac_reward_zero_std": 0.0, "grad_norm": 0.009222477674484253, "learning_rate": 2.0000000000000003e-06, "loss": 0.013532082550227642, "num_tokens": 365240.0, "reward": 0.6518310904502869, "reward_std": 0.3150613605976105, "rewards/correctness/mean": 0.9166666865348816, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.2648356258869171, "rewards/length_penalty/std": 0.0952993780374527, "sampling/importance_sampling_ratio/max": 1.411893367767334, "sampling/importance_sampling_ratio/mean": 0.9938415288925171, "sampling/importance_sampling_ratio/min": 0.6583556532859802, "sampling/sampling_logp_difference/max": 0.41800999641418457, "sampling/sampling_logp_difference/mean": 0.011639133095741272, "step": 7, "step_time": 14.554296029964462 }, { "clip_ratio/high_max": 0.0007130262577751031, "clip_ratio/high_mean": 0.0007130262577751031, "clip_ratio/low_mean": 9.414875239599496e-05, "clip_ratio/low_min": 9.414875239599496e-05, "clip_ratio/region_mean": 0.0008071750247230133, "completions/clipped_ratio": 0.0, "completions/max_length": 1923.0, "completions/max_terminated_length": 1923.0, "completions/mean_length": 596.7000122070312, "completions/mean_terminated_length": 596.7000122070312, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.23006930450598398, "epoch": 0.011661807580174927, "frac_reward_zero_std": 0.0, "grad_norm": 0.009591356851160526, "learning_rate": 2.3333333333333336e-06, "loss": 0.07374510169029236, "num_tokens": 407292.0, "reward": 0.7086426019668579, "reward_std": 0.19280359148979187, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.2913574278354645, "rewards/length_penalty/std": 0.19280359148979187, "sampling/importance_sampling_ratio/max": 1.450048565864563, "sampling/importance_sampling_ratio/mean": 0.991736888885498, "sampling/importance_sampling_ratio/min": 0.6695584058761597, "sampling/sampling_logp_difference/max": 0.4011368751525879, "sampling/sampling_logp_difference/mean": 0.014551110565662384, "step": 8, "step_time": 25.349019592627883 }, { "clip_ratio/high_max": 0.0003552542487644435, "clip_ratio/high_mean": 0.0003552542487644435, "clip_ratio/low_mean": 0.0001425145031438054, "clip_ratio/low_min": 0.0001425145031438054, "clip_ratio/region_mean": 0.0004977687591842065, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1988.0, "completions/mean_length": 907.6834106445312, "completions/mean_terminated_length": 804.0181274414062, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.23283551881710687, "epoch": 0.013119533527696793, "frac_reward_zero_std": 0.0, "grad_norm": 0.011995715089142323, "learning_rate": 2.666666666666667e-06, "loss": -0.005120586603879929, "num_tokens": 466993.0, "reward": 0.04012858122587204, "reward_std": 0.6957505345344543, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.44320476055145264, "rewards/length_penalty/std": 0.28446123003959656, "sampling/importance_sampling_ratio/max": 1.4668807983398438, "sampling/importance_sampling_ratio/mean": 0.9919294118881226, "sampling/importance_sampling_ratio/min": 0.6455317139625549, "sampling/sampling_logp_difference/max": 0.4376809597015381, "sampling/sampling_logp_difference/mean": 0.014371889643371105, "step": 9, "step_time": 27.68978225090541 }, { "clip_ratio/high_max": 0.0007404128070144603, "clip_ratio/high_mean": 0.0007404128070144603, "clip_ratio/low_mean": 0.00014422326057683676, "clip_ratio/low_min": 0.00014422326057683676, "clip_ratio/region_mean": 0.0008846360724419355, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 2032.0, "completions/mean_length": 1016.4500732421875, "completions/mean_terminated_length": 810.1399536132812, "completions/min_length": 385.0, "completions/min_terminated_length": 385.0, "entropy": 0.2652890533208847, "epoch": 0.014577259475218658, "frac_reward_zero_std": 0.0, "grad_norm": 0.01154837105423212, "learning_rate": 3e-06, "loss": 0.05848747491836548, "num_tokens": 534050.0, "reward": 0.17035320401191711, "reward_std": 0.6838861703872681, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.4963134825229645, "rewards/length_penalty/std": 0.26304200291633606, "sampling/importance_sampling_ratio/max": 1.4769505262374878, "sampling/importance_sampling_ratio/mean": 0.9908170104026794, "sampling/importance_sampling_ratio/min": 0.6580171585083008, "sampling/sampling_logp_difference/max": 0.41852426528930664, "sampling/sampling_logp_difference/mean": 0.016183653846383095, "step": 10, "step_time": 28.15277793817222 }, { "clip_ratio/high_max": 0.001398060965584591, "clip_ratio/high_mean": 0.001398060965584591, "clip_ratio/low_mean": 0.0001395033480851756, "clip_ratio/low_min": 0.0001395033480851756, "clip_ratio/region_mean": 0.0015375642785026382, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1751.0, "completions/mean_length": 1006.7500610351562, "completions/mean_terminated_length": 869.2264404296875, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.3573797369996707, "epoch": 0.016034985422740525, "frac_reward_zero_std": 0.0, "grad_norm": 0.016619419679045677, "learning_rate": 3.3333333333333333e-06, "loss": 0.01292102038860321, "num_tokens": 599195.0, "reward": -0.30824384093284607, "reward_std": 0.4079125225543976, "rewards/correctness/mean": 0.18333333730697632, "rewards/correctness/std": 0.39020493626594543, "rewards/length_penalty/mean": -0.4915771484375, "rewards/length_penalty/std": 0.29589328169822693, "sampling/importance_sampling_ratio/max": 1.4458197355270386, "sampling/importance_sampling_ratio/mean": 0.9881013035774231, "sampling/importance_sampling_ratio/min": 0.5618796944618225, "sampling/sampling_logp_difference/max": 0.5764675140380859, "sampling/sampling_logp_difference/mean": 0.019838672131299973, "step": 11, "step_time": 28.012359508080408 }, { "clip_ratio/high_max": 0.0007473976923696076, "clip_ratio/high_mean": 0.0007473976923696076, "clip_ratio/low_mean": 0.0001535463670734316, "clip_ratio/low_min": 0.0001535463670734316, "clip_ratio/region_mean": 0.0009009440545924008, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1887.0, "completions/mean_length": 823.5833740234375, "completions/mean_terminated_length": 661.867919921875, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.28605597962935764, "epoch": 0.01749271137026239, "frac_reward_zero_std": 0.0, "grad_norm": 0.011755434796214104, "learning_rate": 3.6666666666666666e-06, "loss": 0.017311107367277145, "num_tokens": 654090.0, "reward": 0.19785970449447632, "reward_std": 0.6898223757743835, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.4021402895450592, "rewards/length_penalty/std": 0.2941838204860687, "sampling/importance_sampling_ratio/max": 1.4662060737609863, "sampling/importance_sampling_ratio/mean": 0.9898833632469177, "sampling/importance_sampling_ratio/min": 0.6318491697311401, "sampling/sampling_logp_difference/max": 0.4591045379638672, "sampling/sampling_logp_difference/mean": 0.01767781749367714, "step": 12, "step_time": 26.969347662990913 }, { "clip_ratio/high_max": 0.0014183916015705715, "clip_ratio/high_mean": 0.0014183916015705715, "clip_ratio/low_mean": 6.0735284932889044e-05, "clip_ratio/low_min": 6.0735284932889044e-05, "clip_ratio/region_mean": 0.0014791268719515454, "completions/clipped_ratio": 0.0, "completions/max_length": 1254.0, "completions/max_terminated_length": 1254.0, "completions/mean_length": 533.0333862304688, "completions/mean_terminated_length": 533.0333862304688, "completions/min_length": 73.0, "completions/min_terminated_length": 73.0, "entropy": 0.26387997219959897, "epoch": 0.018950437317784258, "frac_reward_zero_std": 0.0, "grad_norm": 0.00885060615837574, "learning_rate": 4.000000000000001e-06, "loss": 0.009860651567578316, "num_tokens": 690932.0, "reward": 0.3063965141773224, "reward_std": 0.5047494769096375, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971747398376465, "rewards/length_penalty/mean": -0.2602701783180237, "rewards/length_penalty/std": 0.14477114379405975, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9907361268997192, "sampling/importance_sampling_ratio/min": 0.580272376537323, "sampling/sampling_logp_difference/max": 1.1491729021072388, "sampling/sampling_logp_difference/mean": 0.01654963009059429, "step": 13, "step_time": 16.872128481976688 }, { "clip_ratio/high_max": 0.0005438411681097932, "clip_ratio/high_mean": 0.0005438411681097932, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0005438411681097932, "completions/clipped_ratio": 0.0, "completions/max_length": 1314.0, "completions/max_terminated_length": 1314.0, "completions/mean_length": 563.433349609375, "completions/mean_terminated_length": 563.433349609375, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.16707361737887064, "epoch": 0.02040816326530612, "frac_reward_zero_std": 0.0, "grad_norm": 0.0070292409509420395, "learning_rate": 4.333333333333334e-06, "loss": 0.01137271523475647, "num_tokens": 728548.0, "reward": 0.3248860836029053, "reward_std": 0.4802418351173401, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.27511394023895264, "rewards/length_penalty/std": 0.10393763333559036, "sampling/importance_sampling_ratio/max": 1.4095503091812134, "sampling/importance_sampling_ratio/mean": 0.9941017031669617, "sampling/importance_sampling_ratio/min": 0.6589340567588806, "sampling/sampling_logp_difference/max": 0.4171319007873535, "sampling/sampling_logp_difference/mean": 0.010680951178073883, "step": 14, "step_time": 16.798709742026404 }, { "clip_ratio/high_max": 0.0009462485904805362, "clip_ratio/high_mean": 0.0009462485904805362, "clip_ratio/low_mean": 0.0001415904177216968, "clip_ratio/low_min": 0.0001415904177216968, "clip_ratio/region_mean": 0.001087839011840212, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 2025.0, "completions/mean_length": 1070.9500732421875, "completions/mean_terminated_length": 875.5399780273438, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.23209517200787863, "epoch": 0.021865889212827987, "frac_reward_zero_std": 0.0, "grad_norm": 0.010992632247507572, "learning_rate": 4.666666666666667e-06, "loss": 0.004310145974159241, "num_tokens": 796855.0, "reward": 0.16040854156017303, "reward_std": 0.6544270515441895, "rewards/correctness/mean": 0.6833333373069763, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.5229247808456421, "rewards/length_penalty/std": 0.26274341344833374, "sampling/importance_sampling_ratio/max": 1.4131360054016113, "sampling/importance_sampling_ratio/mean": 0.9919527769088745, "sampling/importance_sampling_ratio/min": 0.5095058083534241, "sampling/sampling_logp_difference/max": 0.674314022064209, "sampling/sampling_logp_difference/mean": 0.013961256481707096, "step": 15, "step_time": 27.5681957276538 }, { "clip_ratio/high_max": 0.0010235675047927846, "clip_ratio/high_mean": 0.0010235675047927846, "clip_ratio/low_mean": 4.2773429110335805e-05, "clip_ratio/low_min": 4.2773429110335805e-05, "clip_ratio/region_mean": 0.0010663409387537588, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1847.0, "completions/mean_length": 726.7333984375, "completions/mean_terminated_length": 681.1724243164062, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.279613917072614, "epoch": 0.023323615160349854, "frac_reward_zero_std": 0.0, "grad_norm": 0.015736393630504608, "learning_rate": 5e-06, "loss": 0.04625968262553215, "num_tokens": 845189.0, "reward": 0.16181641817092896, "reward_std": 0.5572150945663452, "rewards/correctness/mean": 0.5166666507720947, "rewards/correctness/std": 0.5039393305778503, "rewards/length_penalty/mean": -0.35485026240348816, "rewards/length_penalty/std": 0.18019931018352509, "sampling/importance_sampling_ratio/max": 1.424391269683838, "sampling/importance_sampling_ratio/mean": 0.9905226826667786, "sampling/importance_sampling_ratio/min": 0.676193118095398, "sampling/sampling_logp_difference/max": 0.39127659797668457, "sampling/sampling_logp_difference/mean": 0.016356047242879868, "step": 16, "step_time": 26.450964051065966 }, { "clip_ratio/high_max": 0.001121756814730664, "clip_ratio/high_mean": 0.001121756814730664, "clip_ratio/low_mean": 2.9556067602243274e-05, "clip_ratio/low_min": 2.9556067602243274e-05, "clip_ratio/region_mean": 0.001151312889608865, "completions/clipped_ratio": 0.0, "completions/max_length": 1359.0, "completions/max_terminated_length": 1359.0, "completions/mean_length": 479.35003662109375, "completions/mean_terminated_length": 479.35003662109375, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.23176098863283792, "epoch": 0.02478134110787172, "frac_reward_zero_std": 0.0, "grad_norm": 0.008707236498594284, "learning_rate": 5.333333333333334e-06, "loss": -0.023332105949521065, "num_tokens": 883500.0, "reward": 0.515942394733429, "reward_std": 0.46456581354141235, "rewards/correctness/mean": 0.75, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.23405762016773224, "rewards/length_penalty/std": 0.12105229496955872, "sampling/importance_sampling_ratio/max": 1.437240719795227, "sampling/importance_sampling_ratio/mean": 0.991898775100708, "sampling/importance_sampling_ratio/min": 0.6554322242736816, "sampling/sampling_logp_difference/max": 0.42246031761169434, "sampling/sampling_logp_difference/mean": 0.01451260969042778, "step": 17, "step_time": 18.0038778539747 }, { "clip_ratio/high_max": 0.0006659166829194874, "clip_ratio/high_mean": 0.0006659166829194874, "clip_ratio/low_mean": 0.00017530394082617326, "clip_ratio/low_min": 0.00017530394082617326, "clip_ratio/region_mean": 0.0008412206370849162, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 2021.0, "completions/mean_length": 1156.3333740234375, "completions/mean_terminated_length": 1038.5660400390625, "completions/min_length": 392.0, "completions/min_terminated_length": 392.0, "entropy": 0.2782107988993327, "epoch": 0.026239067055393587, "frac_reward_zero_std": 0.0, "grad_norm": 0.01186887826770544, "learning_rate": 5.666666666666667e-06, "loss": 0.0311305969953537, "num_tokens": 957550.0, "reward": -0.4146158993244171, "reward_std": 0.46414339542388916, "rewards/correctness/mean": 0.15000000596046448, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.5646159052848816, "rewards/length_penalty/std": 0.24389268457889557, "sampling/importance_sampling_ratio/max": 1.4626619815826416, "sampling/importance_sampling_ratio/mean": 0.9902209639549255, "sampling/importance_sampling_ratio/min": 0.6455824971199036, "sampling/sampling_logp_difference/max": 0.43760228157043457, "sampling/sampling_logp_difference/mean": 0.01654873974621296, "step": 18, "step_time": 28.444838002789766 }, { "clip_ratio/high_max": 0.000992356882003757, "clip_ratio/high_mean": 0.000992356882003757, "clip_ratio/low_mean": 3.6043830429359026e-05, "clip_ratio/low_min": 3.6043830429359026e-05, "clip_ratio/region_mean": 0.001028400714858435, "completions/clipped_ratio": 0.0, "completions/max_length": 1005.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 410.4000244140625, "completions/mean_terminated_length": 410.4000244140625, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "entropy": 0.22018084675073624, "epoch": 0.027696793002915453, "frac_reward_zero_std": 0.0, "grad_norm": 0.009298046119511127, "learning_rate": 6e-06, "loss": -0.0038212621584534645, "num_tokens": 985114.0, "reward": 0.5329427123069763, "reward_std": 0.43304309248924255, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.20039062201976776, "rewards/length_penalty/std": 0.09832219034433365, "sampling/importance_sampling_ratio/max": 1.4597746133804321, "sampling/importance_sampling_ratio/mean": 0.9921642541885376, "sampling/importance_sampling_ratio/min": 0.6524658799171448, "sampling/sampling_logp_difference/max": 0.42699646949768066, "sampling/sampling_logp_difference/mean": 0.014085552655160427, "step": 19, "step_time": 13.759585740976036 }, { "clip_ratio/high_max": 0.0007611963568100085, "clip_ratio/high_mean": 0.0007611963568100085, "clip_ratio/low_mean": 0.00018029654286995841, "clip_ratio/low_min": 0.00018029654286995841, "clip_ratio/region_mean": 0.0009414929130192226, "completions/clipped_ratio": 0.18333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 2047.0, "completions/mean_length": 944.2500610351562, "completions/mean_terminated_length": 696.4693603515625, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.2984036107858022, "epoch": 0.029154518950437316, "frac_reward_zero_std": 0.0, "grad_norm": 0.014993884600698948, "learning_rate": 6.333333333333333e-06, "loss": 0.0036488808691501617, "num_tokens": 1045869.0, "reward": -0.1277262419462204, "reward_std": 0.6400496959686279, "rewards/correctness/mean": 0.3333333432674408, "rewards/correctness/std": 0.4753826856613159, "rewards/length_penalty/mean": -0.4610595703125, "rewards/length_penalty/std": 0.329764187335968, "sampling/importance_sampling_ratio/max": 1.7107930183410645, "sampling/importance_sampling_ratio/mean": 0.9897626638412476, "sampling/importance_sampling_ratio/min": 0.5234970450401306, "sampling/sampling_logp_difference/max": 0.647223949432373, "sampling/sampling_logp_difference/mean": 0.017825914546847343, "step": 20, "step_time": 27.421940207248554 }, { "clip_ratio/high_max": 0.0008657800984413674, "clip_ratio/high_mean": 0.0008657800984413674, "clip_ratio/low_mean": 0.00020750967329756045, "clip_ratio/low_min": 0.00020750967329756045, "clip_ratio/region_mean": 0.0010732897547616933, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1960.0, "completions/mean_length": 720.183349609375, "completions/mean_terminated_length": 697.677978515625, "completions/min_length": 164.0, "completions/min_terminated_length": 164.0, "entropy": 0.2126310889919599, "epoch": 0.030612244897959183, "frac_reward_zero_std": 0.0, "grad_norm": 0.011873041279613972, "learning_rate": 6.666666666666667e-06, "loss": 0.022917810827493668, "num_tokens": 1097530.0, "reward": 0.31501466035842896, "reward_std": 0.6336944699287415, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.35165202617645264, "rewards/length_penalty/std": 0.21549133956432343, "sampling/importance_sampling_ratio/max": 1.4036577939987183, "sampling/importance_sampling_ratio/mean": 0.9929825067520142, "sampling/importance_sampling_ratio/min": 0.641616702079773, "sampling/sampling_logp_difference/max": 0.44376420974731445, "sampling/sampling_logp_difference/mean": 0.01318247988820076, "step": 21, "step_time": 28.00862360955216 }, { "clip_ratio/high_max": 0.0009326817137965312, "clip_ratio/high_mean": 0.0009326817137965312, "clip_ratio/low_mean": 2.0387359351540606e-05, "clip_ratio/low_min": 2.0387359351540606e-05, "clip_ratio/region_mean": 0.0009530690828493485, "completions/clipped_ratio": 0.0, "completions/max_length": 1656.0, "completions/max_terminated_length": 1656.0, "completions/mean_length": 784.3833618164062, "completions/mean_terminated_length": 784.3833618164062, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.21914158513148627, "epoch": 0.03206997084548105, "frac_reward_zero_std": 0.0, "grad_norm": 0.01254537794739008, "learning_rate": 7e-06, "loss": -0.0010693036019802094, "num_tokens": 1150133.0, "reward": 0.5503336787223816, "reward_std": 0.34320735931396484, "rewards/correctness/mean": 0.9333333373069763, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.3829996883869171, "rewards/length_penalty/std": 0.18697305023670197, "sampling/importance_sampling_ratio/max": 1.4669926166534424, "sampling/importance_sampling_ratio/mean": 0.9924184679985046, "sampling/importance_sampling_ratio/min": 0.6394065618515015, "sampling/sampling_logp_difference/max": 0.44721484184265137, "sampling/sampling_logp_difference/mean": 0.01387734618037939, "step": 22, "step_time": 22.649415029911324 }, { "clip_ratio/high_max": 0.000687097145904166, "clip_ratio/high_mean": 0.000687097145904166, "clip_ratio/low_mean": 0.00010386897944651234, "clip_ratio/low_min": 0.00010386897944651234, "clip_ratio/region_mean": 0.0007909661168620611, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1801.0, "completions/mean_length": 942.7167358398438, "completions/mean_terminated_length": 884.5438842773438, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.20660034815470377, "epoch": 0.033527696793002916, "frac_reward_zero_std": 0.0, "grad_norm": 0.014352154918015003, "learning_rate": 7.333333333333333e-06, "loss": 0.01802755892276764, "num_tokens": 1211676.0, "reward": -0.21031087636947632, "reward_std": 0.5049320459365845, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.4603108763694763, "rewards/length_penalty/std": 0.2326008826494217, "sampling/importance_sampling_ratio/max": 1.4130291938781738, "sampling/importance_sampling_ratio/mean": 0.9927947521209717, "sampling/importance_sampling_ratio/min": 0.4475542902946472, "sampling/sampling_logp_difference/max": 0.803957462310791, "sampling/sampling_logp_difference/mean": 0.012972496449947357, "step": 23, "step_time": 27.452100809896365 }, { "clip_ratio/high_max": 0.00119293418538291, "clip_ratio/high_mean": 0.00119293418538291, "clip_ratio/low_mean": 0.00015429374737626253, "clip_ratio/low_min": 0.00015429374737626253, "clip_ratio/region_mean": 0.0013472279048680018, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1954.0, "completions/mean_length": 1060.75, "completions/mean_terminated_length": 813.9375, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.3705875674883525, "epoch": 0.03498542274052478, "frac_reward_zero_std": 0.0, "grad_norm": 0.013515997678041458, "learning_rate": 7.666666666666667e-06, "loss": 0.03517736494541168, "num_tokens": 1281111.0, "reward": -0.001277669332921505, "reward_std": 0.719670832157135, "rewards/correctness/mean": 0.5166666507720947, "rewards/correctness/std": 0.5039393305778503, "rewards/length_penalty/mean": -0.5179443359375, "rewards/length_penalty/std": 0.3272433280944824, "sampling/importance_sampling_ratio/max": 1.4604820013046265, "sampling/importance_sampling_ratio/mean": 0.9876757860183716, "sampling/importance_sampling_ratio/min": 0.5807204246520996, "sampling/sampling_logp_difference/max": 0.5434858798980713, "sampling/sampling_logp_difference/mean": 0.020977798849344254, "step": 24, "step_time": 27.973477198043838 }, { "clip_ratio/high_max": 0.0006930709569132887, "clip_ratio/high_mean": 0.0006930709569132887, "clip_ratio/low_mean": 6.285587854411763e-05, "clip_ratio/low_min": 6.285587854411763e-05, "clip_ratio/region_mean": 0.0007559268318194275, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1684.0, "completions/mean_length": 774.4833984375, "completions/mean_terminated_length": 752.8983154296875, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "entropy": 0.18978050351142883, "epoch": 0.03644314868804665, "frac_reward_zero_std": 0.0, "grad_norm": 0.010804434306919575, "learning_rate": 8.000000000000001e-06, "loss": 0.03464656323194504, "num_tokens": 1331990.0, "reward": 0.3385010063648224, "reward_std": 0.5436288714408875, "rewards/correctness/mean": 0.7166666388511658, "rewards/correctness/std": 0.4544196128845215, "rewards/length_penalty/mean": -0.37816569209098816, "rewards/length_penalty/std": 0.204934760928154, "sampling/importance_sampling_ratio/max": 1.4578324556350708, "sampling/importance_sampling_ratio/mean": 0.993187665939331, "sampling/importance_sampling_ratio/min": 0.6711965203285217, "sampling/sampling_logp_difference/max": 0.398693323135376, "sampling/sampling_logp_difference/mean": 0.012119485065340996, "step": 25, "step_time": 25.95817614090629 }, { "clip_ratio/high_max": 0.00038442869602780166, "clip_ratio/high_mean": 0.00038442869602780166, "clip_ratio/low_mean": 0.00017288892801540592, "clip_ratio/low_min": 0.00017288892801540592, "clip_ratio/region_mean": 0.0005573176216178884, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1915.0, "completions/mean_length": 670.9000244140625, "completions/mean_terminated_length": 545.7090454101562, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.16129851341247559, "epoch": 0.037900874635568516, "frac_reward_zero_std": 0.0, "grad_norm": 0.009562763385474682, "learning_rate": 8.333333333333334e-06, "loss": 0.017397675663232803, "num_tokens": 1376984.0, "reward": 0.33907878398895264, "reward_std": 0.6748707294464111, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.32758790254592896, "rewards/length_penalty/std": 0.2830147445201874, "sampling/importance_sampling_ratio/max": 1.4385851621627808, "sampling/importance_sampling_ratio/mean": 0.9943342208862305, "sampling/importance_sampling_ratio/min": 0.5278142690658569, "sampling/sampling_logp_difference/max": 0.6390109062194824, "sampling/sampling_logp_difference/mean": 0.010507638566195965, "step": 26, "step_time": 25.522193972254172 }, { "clip_ratio/high_max": 0.0009138718208608528, "clip_ratio/high_mean": 0.0009138718208608528, "clip_ratio/low_mean": 0.00015279000460092598, "clip_ratio/low_min": 0.00015279000460092598, "clip_ratio/region_mean": 0.0010666618375883747, "completions/clipped_ratio": 0.0, "completions/max_length": 2034.0, "completions/max_terminated_length": 2034.0, "completions/mean_length": 676.6333618164062, "completions/mean_terminated_length": 676.6333618164062, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.23284050822257996, "epoch": 0.03935860058309038, "frac_reward_zero_std": 0.0, "grad_norm": 0.012365774251520634, "learning_rate": 8.666666666666668e-06, "loss": 0.024310387670993805, "num_tokens": 1422522.0, "reward": 0.5196126699447632, "reward_std": 0.4647842049598694, "rewards/correctness/mean": 0.8500000238418579, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.3303873836994171, "rewards/length_penalty/std": 0.17094206809997559, "sampling/importance_sampling_ratio/max": 1.4408317804336548, "sampling/importance_sampling_ratio/mean": 0.9918356537818909, "sampling/importance_sampling_ratio/min": 0.5742970108985901, "sampling/sampling_logp_difference/max": 0.5546085834503174, "sampling/sampling_logp_difference/mean": 0.015102035365998745, "step": 27, "step_time": 24.69438692368567 }, { "clip_ratio/high_max": 0.0010074215630690257, "clip_ratio/high_mean": 0.0010074215630690257, "clip_ratio/low_mean": 6.152293644845486e-05, "clip_ratio/low_min": 6.152293644845486e-05, "clip_ratio/region_mean": 0.0010689445189200342, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1912.0, "completions/mean_length": 750.433349609375, "completions/mean_terminated_length": 705.6896362304688, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.29244403292735416, "epoch": 0.04081632653061224, "frac_reward_zero_std": 0.0, "grad_norm": 0.015244685113430023, "learning_rate": 9e-06, "loss": 0.05068972706794739, "num_tokens": 1471248.0, "reward": 0.30024415254592896, "reward_std": 0.6113126277923584, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.36642253398895264, "rewards/length_penalty/std": 0.24141043424606323, "sampling/importance_sampling_ratio/max": 1.5158343315124512, "sampling/importance_sampling_ratio/mean": 0.9897527694702148, "sampling/importance_sampling_ratio/min": 0.6490998268127441, "sampling/sampling_logp_difference/max": 0.43216872215270996, "sampling/sampling_logp_difference/mean": 0.0177592895925045, "step": 28, "step_time": 25.77531434223056 }, { "clip_ratio/high_max": 0.0013134582453252126, "clip_ratio/high_mean": 0.0013134582453252126, "clip_ratio/low_mean": 0.00016424481145804748, "clip_ratio/low_min": 0.00016424481145804748, "clip_ratio/region_mean": 0.0014777030640592177, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1819.0, "completions/mean_length": 803.2333984375, "completions/mean_terminated_length": 737.7192993164062, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.3203006610274315, "epoch": 0.04227405247813411, "frac_reward_zero_std": 0.0, "grad_norm": 0.012082278728485107, "learning_rate": 9.333333333333334e-06, "loss": 0.011175485327839851, "num_tokens": 1524632.0, "reward": 0.3244629204273224, "reward_std": 0.6027631163597107, "rewards/correctness/mean": 0.7166666388511658, "rewards/correctness/std": 0.4544196128845215, "rewards/length_penalty/mean": -0.39220377802848816, "rewards/length_penalty/std": 0.22215788066387177, "sampling/importance_sampling_ratio/max": 1.4445055723190308, "sampling/importance_sampling_ratio/mean": 0.9887529611587524, "sampling/importance_sampling_ratio/min": 0.6647764444351196, "sampling/sampling_logp_difference/max": 0.40830445289611816, "sampling/sampling_logp_difference/mean": 0.01917846128344536, "step": 29, "step_time": 25.925504898885265 }, { "clip_ratio/high_max": 0.0010785255290102214, "clip_ratio/high_mean": 0.0010785255290102214, "clip_ratio/low_mean": 0.00013830826355842873, "clip_ratio/low_min": 0.00013830826355842873, "clip_ratio/region_mean": 0.0012168337949939694, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1999.0, "completions/mean_length": 915.0000610351562, "completions/mean_terminated_length": 789.1111450195312, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.34353333214918774, "epoch": 0.043731778425655975, "frac_reward_zero_std": 0.0, "grad_norm": 0.014864777214825153, "learning_rate": 9.666666666666667e-06, "loss": 0.005697796121239662, "num_tokens": 1586022.0, "reward": 0.15322266519069672, "reward_std": 0.6906428933143616, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.44677734375, "rewards/length_penalty/std": 0.30850422382354736, "sampling/importance_sampling_ratio/max": 1.4533123970031738, "sampling/importance_sampling_ratio/mean": 0.9885200262069702, "sampling/importance_sampling_ratio/min": 0.547637939453125, "sampling/sampling_logp_difference/max": 0.6021409034729004, "sampling/sampling_logp_difference/mean": 0.01985924318432808, "step": 30, "step_time": 26.621315252035856 }, { "clip_ratio/high_max": 0.001238937091936047, "clip_ratio/high_mean": 0.001238937091936047, "clip_ratio/low_mean": 9.135761744497965e-05, "clip_ratio/low_min": 9.135761744497965e-05, "clip_ratio/region_mean": 0.001330294714231665, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1970.0, "completions/mean_length": 852.5833740234375, "completions/mean_terminated_length": 789.6666870117188, "completions/min_length": 401.0, "completions/min_terminated_length": 401.0, "entropy": 0.2833813379208247, "epoch": 0.04518950437317784, "frac_reward_zero_std": 0.0, "grad_norm": 0.01020822860300541, "learning_rate": 1e-05, "loss": -0.025425449013710022, "num_tokens": 1642917.0, "reward": 0.20036622881889343, "reward_std": 0.5892859697341919, "rewards/correctness/mean": 0.6166666746139526, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.4163004457950592, "rewards/length_penalty/std": 0.2239331156015396, "sampling/importance_sampling_ratio/max": 1.4509061574935913, "sampling/importance_sampling_ratio/mean": 0.9902245998382568, "sampling/importance_sampling_ratio/min": 0.6482113599777222, "sampling/sampling_logp_difference/max": 0.43353843688964844, "sampling/sampling_logp_difference/mean": 0.017317544668912888, "step": 31, "step_time": 26.81202861131169 }, { "clip_ratio/high_max": 0.000777290168722781, "clip_ratio/high_mean": 0.000777290168722781, "clip_ratio/low_mean": 0.00011783391770829137, "clip_ratio/low_min": 0.00011783391770829137, "clip_ratio/region_mean": 0.0008951240840057532, "completions/clipped_ratio": 0.0, "completions/max_length": 1870.0, "completions/max_terminated_length": 1870.0, "completions/mean_length": 869.4667358398438, "completions/mean_terminated_length": 869.4667358398438, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.23462951431671777, "epoch": 0.04664723032069971, "frac_reward_zero_std": 0.0, "grad_norm": 0.010549271479249, "learning_rate": 1.0333333333333335e-05, "loss": -0.00782666727900505, "num_tokens": 1700595.0, "reward": 0.19212239980697632, "reward_std": 0.6066915392875671, "rewards/correctness/mean": 0.6166666746139526, "rewards/correctness/std": 0.4903014302253723, "rewards/length_penalty/mean": -0.4245442748069763, "rewards/length_penalty/std": 0.17661738395690918, "sampling/importance_sampling_ratio/max": 1.4387601613998413, "sampling/importance_sampling_ratio/mean": 0.9920538067817688, "sampling/importance_sampling_ratio/min": 0.6378967761993408, "sampling/sampling_logp_difference/max": 0.44957876205444336, "sampling/sampling_logp_difference/mean": 0.014624694362282753, "step": 32, "step_time": 24.085791839752346 }, { "clip_ratio/high_max": 0.0010240920237265527, "clip_ratio/high_mean": 0.0010240920237265527, "clip_ratio/low_mean": 0.0001451966333358238, "clip_ratio/low_min": 0.0001451966333358238, "clip_ratio/region_mean": 0.0011692886667636533, "completions/clipped_ratio": 0.25, "completions/max_length": 2048.0, "completions/max_terminated_length": 1936.0, "completions/mean_length": 1480.666748046875, "completions/mean_terminated_length": 1291.5555419921875, "completions/min_length": 490.0, "completions/min_terminated_length": 490.0, "entropy": 0.3056724816560745, "epoch": 0.048104956268221574, "frac_reward_zero_std": 0.0, "grad_norm": 0.015298835933208466, "learning_rate": 1.0666666666666667e-05, "loss": 0.06860838085412979, "num_tokens": 1795955.0, "reward": -0.25631511211395264, "reward_std": 0.631487250328064, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.7229817509651184, "rewards/length_penalty/std": 0.22549602389335632, "sampling/importance_sampling_ratio/max": 1.5993895530700684, "sampling/importance_sampling_ratio/mean": 0.9892873167991638, "sampling/importance_sampling_ratio/min": 0.5515028238296509, "sampling/sampling_logp_difference/max": 0.5951082706451416, "sampling/sampling_logp_difference/mean": 0.018112879246473312, "step": 33, "step_time": 30.11426667473279 }, { "clip_ratio/high_max": 0.0005963442818028852, "clip_ratio/high_mean": 0.0005963442818028852, "clip_ratio/low_mean": 8.18812283493268e-05, "clip_ratio/low_min": 8.18812283493268e-05, "clip_ratio/region_mean": 0.0006782255028762544, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1275.0, "completions/mean_length": 730.4000244140625, "completions/mean_terminated_length": 584.0, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.2809029494722684, "epoch": 0.04956268221574344, "frac_reward_zero_std": 0.0, "grad_norm": 0.012408322654664516, "learning_rate": 1.1000000000000001e-05, "loss": 0.055800504982471466, "num_tokens": 1845059.0, "reward": 0.22669272124767303, "reward_std": 0.6471343636512756, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.35664063692092896, "rewards/length_penalty/std": 0.2492990493774414, "sampling/importance_sampling_ratio/max": 1.6343612670898438, "sampling/importance_sampling_ratio/mean": 0.9900498986244202, "sampling/importance_sampling_ratio/min": 0.6331878900527954, "sampling/sampling_logp_difference/max": 0.491252064704895, "sampling/sampling_logp_difference/mean": 0.017236072570085526, "step": 34, "step_time": 25.358212631428614 }, { "clip_ratio/high_max": 0.0007682585661920408, "clip_ratio/high_mean": 0.0007682585661920408, "clip_ratio/low_mean": 0.0001700489034798617, "clip_ratio/low_min": 0.0001700489034798617, "clip_ratio/region_mean": 0.0009383074842238178, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1947.0, "completions/mean_length": 904.5167236328125, "completions/mean_terminated_length": 777.4629516601562, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.24102867394685745, "epoch": 0.05102040816326531, "frac_reward_zero_std": 0.0, "grad_norm": 0.0077178627252578735, "learning_rate": 1.1333333333333334e-05, "loss": -0.007520838174968958, "num_tokens": 1903950.0, "reward": 0.0916748121380806, "reward_std": 0.6451828479766846, "rewards/correctness/mean": 0.5333333611488342, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.44165852665901184, "rewards/length_penalty/std": 0.277593731880188, "sampling/importance_sampling_ratio/max": 1.4403471946716309, "sampling/importance_sampling_ratio/mean": 0.9910374283790588, "sampling/importance_sampling_ratio/min": 0.6097314953804016, "sampling/sampling_logp_difference/max": 0.49473655223846436, "sampling/sampling_logp_difference/mean": 0.015411383472383022, "step": 35, "step_time": 26.316329368855804 }, { "clip_ratio/high_max": 0.0007903170286832998, "clip_ratio/high_mean": 0.0007903170286832998, "clip_ratio/low_mean": 2.127206971636042e-05, "clip_ratio/low_min": 2.127206971636042e-05, "clip_ratio/region_mean": 0.000811589095974341, "completions/clipped_ratio": 0.0, "completions/max_length": 1110.0, "completions/max_terminated_length": 1110.0, "completions/mean_length": 726.1500244140625, "completions/mean_terminated_length": 726.1500244140625, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.15926587333281836, "epoch": 0.052478134110787174, "frac_reward_zero_std": 0.0, "grad_norm": 0.007803198881447315, "learning_rate": 1.1666666666666668e-05, "loss": -0.009670584462583065, "num_tokens": 1952089.0, "reward": 0.4287679195404053, "reward_std": 0.4443022906780243, "rewards/correctness/mean": 0.7833333611488342, "rewards/correctness/std": 0.41545024514198303, "rewards/length_penalty/mean": -0.35456544160842896, "rewards/length_penalty/std": 0.11235044151544571, "sampling/importance_sampling_ratio/max": 1.724177360534668, "sampling/importance_sampling_ratio/mean": 0.9943934082984924, "sampling/importance_sampling_ratio/min": 0.6505312323570251, "sampling/sampling_logp_difference/max": 0.5447499752044678, "sampling/sampling_logp_difference/mean": 0.010538090020418167, "step": 36, "step_time": 15.591551100835204 }, { "clip_ratio/high_max": 0.0010752740296690415, "clip_ratio/high_mean": 0.0010752740296690415, "clip_ratio/low_mean": 2.604980788116033e-05, "clip_ratio/low_min": 2.604980788116033e-05, "clip_ratio/region_mean": 0.001101323839975521, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1881.0, "completions/mean_length": 903.1666870117188, "completions/mean_terminated_length": 842.9122924804688, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.27130760500828427, "epoch": 0.05393586005830904, "frac_reward_zero_std": 0.0, "grad_norm": 0.014049148187041283, "learning_rate": 1.2e-05, "loss": 0.058295562863349915, "num_tokens": 2010369.0, "reward": 0.25900065898895264, "reward_std": 0.4945225119590759, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.462124764919281, "rewards/length_penalty/mean": -0.4409993588924408, "rewards/length_penalty/std": 0.25441300868988037, "sampling/importance_sampling_ratio/max": 1.4194098711013794, "sampling/importance_sampling_ratio/mean": 0.9906554222106934, "sampling/importance_sampling_ratio/min": 0.5438868403434753, "sampling/sampling_logp_difference/max": 0.6090140342712402, "sampling/sampling_logp_difference/mean": 0.016203705221414566, "step": 37, "step_time": 25.97705299197696 }, { "clip_ratio/high_max": 0.0008416666726892194, "clip_ratio/high_mean": 0.0008416666726892194, "clip_ratio/low_mean": 0.00015480869236246994, "clip_ratio/low_min": 0.00015480869236246994, "clip_ratio/region_mean": 0.0009964753990061581, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1893.0, "completions/mean_length": 1027.61669921875, "completions/mean_terminated_length": 954.732177734375, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.2630870093901952, "epoch": 0.05539358600583091, "frac_reward_zero_std": 0.0, "grad_norm": 0.013990028761327267, "learning_rate": 1.2333333333333334e-05, "loss": 0.048009540885686874, "num_tokens": 2077336.0, "reward": -0.01843261905014515, "reward_std": 0.6465713381767273, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.5017659664154053, "rewards/length_penalty/std": 0.23699496686458588, "sampling/importance_sampling_ratio/max": 1.5592713356018066, "sampling/importance_sampling_ratio/mean": 0.9906838536262512, "sampling/importance_sampling_ratio/min": 0.4875663220882416, "sampling/sampling_logp_difference/max": 0.7183289527893066, "sampling/sampling_logp_difference/mean": 0.016107842326164246, "step": 38, "step_time": 27.35881438339129 }, { "clip_ratio/high_max": 0.000449634826509282, "clip_ratio/high_mean": 0.000449634826509282, "clip_ratio/low_mean": 8.750285148077334e-05, "clip_ratio/low_min": 8.750285148077334e-05, "clip_ratio/region_mean": 0.0005371376804153746, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1747.0, "completions/mean_length": 681.7166748046875, "completions/mean_terminated_length": 658.559326171875, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.1788865476846695, "epoch": 0.056851311953352766, "frac_reward_zero_std": 0.0, "grad_norm": 0.011713888496160507, "learning_rate": 1.2666666666666667e-05, "loss": -0.010897926986217499, "num_tokens": 2122019.0, "reward": 0.40046387910842896, "reward_std": 0.5082594156265259, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459485113620758, "rewards/length_penalty/mean": -0.3328694701194763, "rewards/length_penalty/std": 0.19904670119285583, "sampling/importance_sampling_ratio/max": 1.467110276222229, "sampling/importance_sampling_ratio/mean": 0.9937765598297119, "sampling/importance_sampling_ratio/min": 0.6686986684799194, "sampling/sampling_logp_difference/max": 0.4024217128753662, "sampling/sampling_logp_difference/mean": 0.011745237745344639, "step": 39, "step_time": 25.736718475585803 }, { "clip_ratio/high_max": 0.0010374786409859855, "clip_ratio/high_mean": 0.0010374786409859855, "clip_ratio/low_mean": 9.631955617805943e-05, "clip_ratio/low_min": 9.631955617805943e-05, "clip_ratio/region_mean": 0.0011337981753361721, "completions/clipped_ratio": 0.0, "completions/max_length": 1461.0, "completions/max_terminated_length": 1461.0, "completions/mean_length": 674.300048828125, "completions/mean_terminated_length": 674.300048828125, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.18291178345680237, "epoch": 0.05830903790087463, "frac_reward_zero_std": 0.0, "grad_norm": 0.00884868111461401, "learning_rate": 1.3000000000000001e-05, "loss": 0.001374863088130951, "num_tokens": 2167937.0, "reward": 0.520751953125, "reward_std": 0.39113694429397583, "rewards/correctness/mean": 0.8500000238418579, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.3292480409145355, "rewards/length_penalty/std": 0.1301271766424179, "sampling/importance_sampling_ratio/max": 1.3339730501174927, "sampling/importance_sampling_ratio/mean": 0.9936630725860596, "sampling/importance_sampling_ratio/min": 0.6585886478424072, "sampling/sampling_logp_difference/max": 0.41765618324279785, "sampling/sampling_logp_difference/mean": 0.011697357520461082, "step": 40, "step_time": 18.73584145028144 }, { "clip_ratio/high_max": 0.000907802406193999, "clip_ratio/high_mean": 0.000907802406193999, "clip_ratio/low_mean": 0.00012981869319143394, "clip_ratio/low_min": 0.00012981869319143394, "clip_ratio/region_mean": 0.0010376211042360712, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1965.0, "completions/mean_length": 874.4166870117188, "completions/mean_terminated_length": 812.6491088867188, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.1831328049302101, "epoch": 0.0597667638483965, "frac_reward_zero_std": 0.0, "grad_norm": 0.011611186899244785, "learning_rate": 1.3333333333333333e-05, "loss": 0.0037883776240050793, "num_tokens": 2225772.0, "reward": 0.3063720762729645, "reward_std": 0.6729326844215393, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.4269612729549408, "rewards/length_penalty/std": 0.2836941182613373, "sampling/importance_sampling_ratio/max": 1.8383829593658447, "sampling/importance_sampling_ratio/mean": 0.9938834309577942, "sampling/importance_sampling_ratio/min": 0.6318473219871521, "sampling/sampling_logp_difference/max": 0.6088863611221313, "sampling/sampling_logp_difference/mean": 0.01193917915225029, "step": 41, "step_time": 26.612672707997262 }, { "clip_ratio/high_max": 0.0006895982175289342, "clip_ratio/high_mean": 0.0006895982175289342, "clip_ratio/low_mean": 0.00018824153206272362, "clip_ratio/low_min": 0.00018824153206272362, "clip_ratio/region_mean": 0.0008778397411030406, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 2027.0, "completions/mean_length": 1014.8500366210938, "completions/mean_terminated_length": 808.219970703125, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.3642863258719444, "epoch": 0.061224489795918366, "frac_reward_zero_std": 0.0, "grad_norm": 0.016840942203998566, "learning_rate": 1.3666666666666667e-05, "loss": 0.03785286471247673, "num_tokens": 2292483.0, "reward": -0.02886556088924408, "reward_std": 0.7496117949485779, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.49553221464157104, "rewards/length_penalty/std": 0.32649514079093933, "sampling/importance_sampling_ratio/max": 1.4769392013549805, "sampling/importance_sampling_ratio/mean": 0.9869816303253174, "sampling/importance_sampling_ratio/min": 0.07880815118551254, "sampling/sampling_logp_difference/max": 2.540738821029663, "sampling/sampling_logp_difference/mean": 0.021654432639479637, "step": 42, "step_time": 27.032590586226434 }, { "clip_ratio/high_max": 0.0006048179056961089, "clip_ratio/high_mean": 0.0006048179056961089, "clip_ratio/low_mean": 0.00022036849622963928, "clip_ratio/low_min": 0.00022036849622963928, "clip_ratio/region_mean": 0.0008251863958624502, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1368.0, "completions/mean_length": 881.0667114257812, "completions/mean_terminated_length": 647.6799926757812, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.3163422793149948, "epoch": 0.06268221574344024, "frac_reward_zero_std": 0.0, "grad_norm": 0.0103360740467906, "learning_rate": 1.4e-05, "loss": 0.0021490827202796936, "num_tokens": 2350457.0, "reward": 0.3031250238418579, "reward_std": 0.6805951595306396, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.43020832538604736, "rewards/length_penalty/std": 0.27624472975730896, "sampling/importance_sampling_ratio/max": 1.4635086059570312, "sampling/importance_sampling_ratio/mean": 0.988986074924469, "sampling/importance_sampling_ratio/min": 0.60587477684021, "sampling/sampling_logp_difference/max": 0.5010819435119629, "sampling/sampling_logp_difference/mean": 0.019041329622268677, "step": 43, "step_time": 26.36771490587853 }, { "clip_ratio/high_max": 0.000841226331734409, "clip_ratio/high_mean": 0.000841226331734409, "clip_ratio/low_mean": 2.286864279691751e-05, "clip_ratio/low_min": 2.286864279691751e-05, "clip_ratio/region_mean": 0.0008640949769566456, "completions/clipped_ratio": 0.0, "completions/max_length": 1618.0, "completions/max_terminated_length": 1618.0, "completions/mean_length": 703.7166748046875, "completions/mean_terminated_length": 703.7166748046875, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.179917444785436, "epoch": 0.0641399416909621, "frac_reward_zero_std": 0.0, "grad_norm": 0.008084164001047611, "learning_rate": 1.4333333333333334e-05, "loss": 0.00908623356372118, "num_tokens": 2397870.0, "reward": 0.23972168564796448, "reward_std": 0.46271032094955444, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.3436116576194763, "rewards/length_penalty/std": 0.16231344640254974, "sampling/importance_sampling_ratio/max": 1.4407228231430054, "sampling/importance_sampling_ratio/mean": 0.9936022758483887, "sampling/importance_sampling_ratio/min": 0.6691447496414185, "sampling/sampling_logp_difference/max": 0.40175485610961914, "sampling/sampling_logp_difference/mean": 0.011499114334583282, "step": 44, "step_time": 20.7415090627037 }, { "clip_ratio/high_max": 0.00041327955599020544, "clip_ratio/high_mean": 0.00041327955599020544, "clip_ratio/low_mean": 0.00023016609460076629, "clip_ratio/low_min": 0.00023016609460076629, "clip_ratio/region_mean": 0.0006434456445276737, "completions/clipped_ratio": 0.3333333432674408, "completions/max_length": 2048.0, "completions/max_terminated_length": 1950.0, "completions/mean_length": 1233.0667724609375, "completions/mean_terminated_length": 825.6000366210938, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.2690398320555687, "epoch": 0.06559766763848396, "frac_reward_zero_std": 0.0, "grad_norm": 0.014921208843588829, "learning_rate": 1.4666666666666666e-05, "loss": 0.05788666009902954, "num_tokens": 2477384.0, "reward": -0.15208333730697632, "reward_std": 0.7774261832237244, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.6020833253860474, "rewards/length_penalty/std": 0.33607375621795654, "sampling/importance_sampling_ratio/max": 1.631103754043579, "sampling/importance_sampling_ratio/mean": 0.9906483292579651, "sampling/importance_sampling_ratio/min": 0.6383159160614014, "sampling/sampling_logp_difference/max": 0.48925697803497314, "sampling/sampling_logp_difference/mean": 0.01613265834748745, "step": 45, "step_time": 28.280902277911082 }, { "clip_ratio/high_max": 0.0004557327993097715, "clip_ratio/high_mean": 0.0004557327993097715, "clip_ratio/low_mean": 8.34783665292586e-05, "clip_ratio/low_min": 8.34783665292586e-05, "clip_ratio/region_mean": 0.0005392111658390301, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1689.0, "completions/mean_length": 601.0667114257812, "completions/mean_terminated_length": 576.5423583984375, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.2017199049393336, "epoch": 0.06705539358600583, "frac_reward_zero_std": 0.0, "grad_norm": 0.008424344472587109, "learning_rate": 1.5000000000000002e-05, "loss": 0.04910995066165924, "num_tokens": 2516748.0, "reward": 0.37317711114883423, "reward_std": 0.6525160074234009, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.29348957538604736, "rewards/length_penalty/std": 0.19386924803256989, "sampling/importance_sampling_ratio/max": 1.410577416419983, "sampling/importance_sampling_ratio/mean": 0.9931090474128723, "sampling/importance_sampling_ratio/min": 0.6503152251243591, "sampling/sampling_logp_difference/max": 0.4302980899810791, "sampling/sampling_logp_difference/mean": 0.012660418637096882, "step": 46, "step_time": 24.2268779524602 }, { "clip_ratio/high_max": 0.000743589373693491, "clip_ratio/high_mean": 0.000743589373693491, "clip_ratio/low_mean": 9.949510179770489e-05, "clip_ratio/low_min": 9.949510179770489e-05, "clip_ratio/region_mean": 0.0008430844754911959, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1772.0, "completions/mean_length": 649.2333374023438, "completions/mean_terminated_length": 575.614013671875, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.2584753731886546, "epoch": 0.06851311953352769, "frac_reward_zero_std": 0.0, "grad_norm": 0.017546163871884346, "learning_rate": 1.5333333333333334e-05, "loss": 0.017078937962651253, "num_tokens": 2560162.0, "reward": 0.43299156427383423, "reward_std": 0.5498223900794983, "rewards/correctness/mean": 0.75, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.31700846552848816, "rewards/length_penalty/std": 0.2566172182559967, "sampling/importance_sampling_ratio/max": 1.413201093673706, "sampling/importance_sampling_ratio/mean": 0.9907872080802917, "sampling/importance_sampling_ratio/min": 0.5558434724807739, "sampling/sampling_logp_difference/max": 0.587268590927124, "sampling/sampling_logp_difference/mean": 0.016103271394968033, "step": 47, "step_time": 25.70594487292692 }, { "clip_ratio/high_max": 0.0008645804336993024, "clip_ratio/high_mean": 0.0008645804336993024, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008645804336993024, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1680.0, "completions/mean_length": 594.2333374023438, "completions/mean_terminated_length": 569.5932006835938, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.24059434731801352, "epoch": 0.06997084548104957, "frac_reward_zero_std": 0.0, "grad_norm": 0.015697062015533447, "learning_rate": 1.5666666666666667e-05, "loss": 0.027538156136870384, "num_tokens": 2600006.0, "reward": 0.20984701812267303, "reward_std": 0.5217742323875427, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.29015299677848816, "rewards/length_penalty/std": 0.18181516230106354, "sampling/importance_sampling_ratio/max": 1.3966842889785767, "sampling/importance_sampling_ratio/mean": 0.9921780824661255, "sampling/importance_sampling_ratio/min": 0.6649665236473083, "sampling/sampling_logp_difference/max": 0.4080185890197754, "sampling/sampling_logp_difference/mean": 0.01451186928898096, "step": 48, "step_time": 24.447830377845094 }, { "clip_ratio/high_max": 0.00046326468388239544, "clip_ratio/high_mean": 0.00046326468388239544, "clip_ratio/low_mean": 0.00016435115200389797, "clip_ratio/low_min": 0.00016435115200389797, "clip_ratio/region_mean": 0.0006276158189090589, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1133.0, "completions/mean_length": 634.2166748046875, "completions/mean_terminated_length": 585.4655151367188, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.22726024935642877, "epoch": 0.07142857142857142, "frac_reward_zero_std": 0.0, "grad_norm": 0.010333630256354809, "learning_rate": 1.6000000000000003e-05, "loss": 0.07413791865110397, "num_tokens": 2641989.0, "reward": 0.19032390415668488, "reward_std": 0.5556463599205017, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.3096761107444763, "rewards/length_penalty/std": 0.17995549738407135, "sampling/importance_sampling_ratio/max": 1.4583498239517212, "sampling/importance_sampling_ratio/mean": 0.9922860264778137, "sampling/importance_sampling_ratio/min": 0.645832359790802, "sampling/sampling_logp_difference/max": 0.43721532821655273, "sampling/sampling_logp_difference/mean": 0.01424582302570343, "step": 49, "step_time": 25.561362505191937 }, { "clip_ratio/high_max": 0.0008010070159798488, "clip_ratio/high_mean": 0.0008010070159798488, "clip_ratio/low_mean": 5.733677365545494e-05, "clip_ratio/low_min": 5.733677365545494e-05, "clip_ratio/region_mean": 0.0008583437884226441, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1806.0, "completions/mean_length": 769.5333862304688, "completions/mean_terminated_length": 702.24560546875, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.22106604278087616, "epoch": 0.0728862973760933, "frac_reward_zero_std": 0.0, "grad_norm": 0.011128261685371399, "learning_rate": 1.6333333333333335e-05, "loss": 0.04742317646741867, "num_tokens": 2692871.0, "reward": 0.20758464932441711, "reward_std": 0.6197415590286255, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.3757486939430237, "rewards/length_penalty/std": 0.23703652620315552, "sampling/importance_sampling_ratio/max": 1.5202347040176392, "sampling/importance_sampling_ratio/mean": 0.9924477934837341, "sampling/importance_sampling_ratio/min": 0.6783851981163025, "sampling/sampling_logp_difference/max": 0.41886472702026367, "sampling/sampling_logp_difference/mean": 0.013558726757764816, "step": 50, "step_time": 25.64299608883448 }, { "clip_ratio/high_max": 0.000875128636835143, "clip_ratio/high_mean": 0.000875128636835143, "clip_ratio/low_mean": 0.0001807594186781595, "clip_ratio/low_min": 0.0001807594186781595, "clip_ratio/region_mean": 0.0010558880652145792, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1684.0, "completions/mean_length": 744.5000610351562, "completions/mean_terminated_length": 699.5516967773438, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.28909573952356976, "epoch": 0.07434402332361516, "frac_reward_zero_std": 0.0, "grad_norm": 0.021083198487758636, "learning_rate": 1.6666666666666667e-05, "loss": 0.06212671101093292, "num_tokens": 2743481.0, "reward": 0.11980795115232468, "reward_std": 0.61191326379776, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.363525390625, "rewards/length_penalty/std": 0.22039349377155304, "sampling/importance_sampling_ratio/max": 1.4389187097549438, "sampling/importance_sampling_ratio/mean": 0.9900707006454468, "sampling/importance_sampling_ratio/min": 0.5461902618408203, "sampling/sampling_logp_difference/max": 0.6047878265380859, "sampling/sampling_logp_difference/mean": 0.017100242897868156, "step": 51, "step_time": 26.10056390124373 }, { "clip_ratio/high_max": 0.0007763957012988006, "clip_ratio/high_mean": 0.0007763957012988006, "clip_ratio/low_mean": 0.00010222260607406497, "clip_ratio/low_min": 0.00010222260607406497, "clip_ratio/region_mean": 0.000878618312223504, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1858.0, "completions/mean_length": 867.300048828125, "completions/mean_terminated_length": 711.3585205078125, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.2560170814394951, "epoch": 0.07580174927113703, "frac_reward_zero_std": 0.0, "grad_norm": 0.013316703028976917, "learning_rate": 1.7e-05, "loss": 0.01976807788014412, "num_tokens": 2800329.0, "reward": 0.4098470211029053, "reward_std": 0.5818691253662109, "rewards/correctness/mean": 0.8333333134651184, "rewards/correctness/std": 0.3758230209350586, "rewards/length_penalty/mean": -0.4234863221645355, "rewards/length_penalty/std": 0.2742421329021454, "sampling/importance_sampling_ratio/max": 1.9864535331726074, "sampling/importance_sampling_ratio/mean": 0.990938663482666, "sampling/importance_sampling_ratio/min": 0.6354402899742126, "sampling/sampling_logp_difference/max": 0.68635094165802, "sampling/sampling_logp_difference/mean": 0.01604917272925377, "step": 52, "step_time": 26.209189630812034 }, { "clip_ratio/high_max": 0.0006766375542307893, "clip_ratio/high_mean": 0.0006766375542307893, "clip_ratio/low_mean": 0.00015319734544997723, "clip_ratio/low_min": 0.00015319734544997723, "clip_ratio/region_mean": 0.000829834898468107, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 1797.0, "completions/mean_length": 1292.0001220703125, "completions/mean_terminated_length": 1017.0909423828125, "completions/min_length": 489.0, "completions/min_terminated_length": 489.0, "entropy": 0.30388736476500827, "epoch": 0.07725947521865889, "frac_reward_zero_std": 0.0, "grad_norm": 0.011573869735002518, "learning_rate": 1.7333333333333336e-05, "loss": -0.0076588112860918045, "num_tokens": 2882579.0, "reward": -0.16419272124767303, "reward_std": 0.6786981821060181, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.630859375, "rewards/length_penalty/std": 0.26540708541870117, "sampling/importance_sampling_ratio/max": 1.695695400238037, "sampling/importance_sampling_ratio/mean": 0.989521861076355, "sampling/importance_sampling_ratio/min": 0.6508986353874207, "sampling/sampling_logp_difference/max": 0.5280928611755371, "sampling/sampling_logp_difference/mean": 0.017895212396979332, "step": 53, "step_time": 28.47213218314573 }, { "clip_ratio/high_max": 0.000526597427475887, "clip_ratio/high_mean": 0.000526597427475887, "clip_ratio/low_mean": 5.23866571408386e-05, "clip_ratio/low_min": 5.23866571408386e-05, "clip_ratio/region_mean": 0.0005789840773407681, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 2012.0, "completions/mean_length": 897.1000366210938, "completions/mean_terminated_length": 857.413818359375, "completions/min_length": 416.0, "completions/min_terminated_length": 416.0, "entropy": 0.2398771271109581, "epoch": 0.07871720116618076, "frac_reward_zero_std": 0.0, "grad_norm": 0.012547567486763, "learning_rate": 1.7666666666666668e-05, "loss": 0.000821845605969429, "num_tokens": 2940715.0, "reward": -0.0713704451918602, "reward_std": 0.5608754754066467, "rewards/correctness/mean": 0.36666667461395264, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.43803709745407104, "rewards/length_penalty/std": 0.21056923270225525, "sampling/importance_sampling_ratio/max": 1.5159095525741577, "sampling/importance_sampling_ratio/mean": 0.991970419883728, "sampling/importance_sampling_ratio/min": 0.6778066754341125, "sampling/sampling_logp_difference/max": 0.416015625, "sampling/sampling_logp_difference/mean": 0.014678750187158585, "step": 54, "step_time": 25.951356765115634 }, { "clip_ratio/high_max": 0.000771099810663145, "clip_ratio/high_mean": 0.000771099810663145, "clip_ratio/low_mean": 0.0002113857032478942, "clip_ratio/low_min": 0.0002113857032478942, "clip_ratio/region_mean": 0.0009824855321009334, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1933.0, "completions/mean_length": 938.9833984375, "completions/mean_terminated_length": 717.1799926757812, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.32161477704842883, "epoch": 0.08017492711370262, "frac_reward_zero_std": 0.0, "grad_norm": 0.011260569095611572, "learning_rate": 1.8e-05, "loss": 0.01119990274310112, "num_tokens": 3000564.0, "reward": 0.04151204600930214, "reward_std": 0.7691377997398376, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.45848795771598816, "rewards/length_penalty/std": 0.3181942105293274, "sampling/importance_sampling_ratio/max": 1.4323796033859253, "sampling/importance_sampling_ratio/mean": 0.9886839985847473, "sampling/importance_sampling_ratio/min": 0.6530927419662476, "sampling/sampling_logp_difference/max": 0.42603611946105957, "sampling/sampling_logp_difference/mean": 0.019134491682052612, "step": 55, "step_time": 26.1115862026345 }, { "clip_ratio/high_max": 0.0004554060918356602, "clip_ratio/high_mean": 0.0004554060918356602, "clip_ratio/low_mean": 0.00012906974249441797, "clip_ratio/low_min": 0.00012906974249441797, "clip_ratio/region_mean": 0.0005844758367553974, "completions/clipped_ratio": 0.25, "completions/max_length": 2048.0, "completions/max_terminated_length": 1948.0, "completions/mean_length": 1060.9833984375, "completions/mean_terminated_length": 731.977783203125, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "entropy": 0.22106081744035086, "epoch": 0.08163265306122448, "frac_reward_zero_std": 0.0, "grad_norm": 0.010457084514200687, "learning_rate": 1.8333333333333333e-05, "loss": -0.008733715862035751, "num_tokens": 3070233.0, "reward": -0.25139161944389343, "reward_std": 0.6637234091758728, "rewards/correctness/mean": 0.2666666805744171, "rewards/correctness/std": 0.4459485113620758, "rewards/length_penalty/mean": -0.5180582404136658, "rewards/length_penalty/std": 0.32603931427001953, "sampling/importance_sampling_ratio/max": 1.4403032064437866, "sampling/importance_sampling_ratio/mean": 0.9919553995132446, "sampling/importance_sampling_ratio/min": 0.5502684712409973, "sampling/sampling_logp_difference/max": 0.5973489284515381, "sampling/sampling_logp_difference/mean": 0.014011329971253872, "step": 56, "step_time": 27.72089922009036 }, { "clip_ratio/high_max": 0.00037454012514596496, "clip_ratio/high_mean": 0.00037454012514596496, "clip_ratio/low_mean": 0.00010635672030427183, "clip_ratio/low_min": 0.00010635672030427183, "clip_ratio/region_mean": 0.00048089684059959836, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 726.2667236328125, "completions/mean_terminated_length": 493.0196228027344, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "entropy": 0.24900235484043756, "epoch": 0.08309037900874636, "frac_reward_zero_std": 0.0, "grad_norm": 0.01727903261780739, "learning_rate": 1.866666666666667e-05, "loss": 0.00045283790677785873, "num_tokens": 3118369.0, "reward": 0.49537762999534607, "reward_std": 0.649752676486969, "rewards/correctness/mean": 0.8500000238418579, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.35462239384651184, "rewards/length_penalty/std": 0.30922913551330566, "sampling/importance_sampling_ratio/max": 1.467110276222229, "sampling/importance_sampling_ratio/mean": 0.9905803799629211, "sampling/importance_sampling_ratio/min": 0.3542850613594055, "sampling/sampling_logp_difference/max": 1.0376534461975098, "sampling/sampling_logp_difference/mean": 0.015638122335076332, "step": 57, "step_time": 25.36844819993712 }, { "clip_ratio/high_max": 0.0007073824381222948, "clip_ratio/high_mean": 0.0007073824381222948, "clip_ratio/low_mean": 2.5406504088702302e-05, "clip_ratio/low_min": 2.5406504088702302e-05, "clip_ratio/region_mean": 0.0007327889470616356, "completions/clipped_ratio": 0.0, "completions/max_length": 1676.0, "completions/max_terminated_length": 1676.0, "completions/mean_length": 591.0667114257812, "completions/mean_terminated_length": 591.0667114257812, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "entropy": 0.2171641836563746, "epoch": 0.08454810495626822, "frac_reward_zero_std": 0.0, "grad_norm": 0.010724866762757301, "learning_rate": 1.9e-05, "loss": 0.006157484836876392, "num_tokens": 3158633.0, "reward": 0.3447265923023224, "reward_std": 0.614934504032135, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.28860676288604736, "rewards/length_penalty/std": 0.18544556200504303, "sampling/importance_sampling_ratio/max": 1.6070520877838135, "sampling/importance_sampling_ratio/mean": 0.992554247379303, "sampling/importance_sampling_ratio/min": 0.561149001121521, "sampling/sampling_logp_difference/max": 0.5777688026428223, "sampling/sampling_logp_difference/mean": 0.013930361717939377, "step": 58, "step_time": 20.642161039169878 }, { "clip_ratio/high_max": 0.0011212155416918297, "clip_ratio/high_mean": 0.0011212155416918297, "clip_ratio/low_mean": 0.0001151398682850413, "clip_ratio/low_min": 0.0001151398682850413, "clip_ratio/region_mean": 0.0012363554366553824, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1823.0, "completions/mean_length": 890.3833618164062, "completions/mean_terminated_length": 785.1454467773438, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.2747861370444298, "epoch": 0.08600583090379009, "frac_reward_zero_std": 0.0, "grad_norm": 0.018310362473130226, "learning_rate": 1.9333333333333333e-05, "loss": 0.03689084202051163, "num_tokens": 3217516.0, "reward": -0.1680908352136612, "reward_std": 0.5773100852966309, "rewards/correctness/mean": 0.2666666805744171, "rewards/correctness/std": 0.4459485113620758, "rewards/length_penalty/mean": -0.4347575008869171, "rewards/length_penalty/std": 0.2805073857307434, "sampling/importance_sampling_ratio/max": 1.4319807291030884, "sampling/importance_sampling_ratio/mean": 0.9904412031173706, "sampling/importance_sampling_ratio/min": 0.5849228501319885, "sampling/sampling_logp_difference/max": 0.5362752676010132, "sampling/sampling_logp_difference/mean": 0.0173331331461668, "step": 59, "step_time": 27.54057458974421 }, { "clip_ratio/high_max": 0.001425526337698102, "clip_ratio/high_mean": 0.001425526337698102, "clip_ratio/low_mean": 0.00010533920916107793, "clip_ratio/low_min": 0.00010533920916107793, "clip_ratio/region_mean": 0.0015308655371579032, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 2044.0, "completions/mean_length": 776.11669921875, "completions/mean_terminated_length": 754.559326171875, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.2769348993897438, "epoch": 0.08746355685131195, "frac_reward_zero_std": 0.0, "grad_norm": 0.014079735614359379, "learning_rate": 1.9666666666666666e-05, "loss": 0.012713806703686714, "num_tokens": 3267503.0, "reward": 0.30437013506889343, "reward_std": 0.501975417137146, "rewards/correctness/mean": 0.6833333373069763, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.3789632022380829, "rewards/length_penalty/std": 0.2151976078748703, "sampling/importance_sampling_ratio/max": 1.6477974653244019, "sampling/importance_sampling_ratio/mean": 0.9902998805046082, "sampling/importance_sampling_ratio/min": 0.5580717325210571, "sampling/sampling_logp_difference/max": 0.5832676887512207, "sampling/sampling_logp_difference/mean": 0.0172236617654562, "step": 60, "step_time": 25.26443952252157 }, { "clip_ratio/high_max": 0.0006640923747909255, "clip_ratio/high_mean": 0.0006640923747909255, "clip_ratio/low_mean": 6.594129323881741e-05, "clip_ratio/low_min": 6.594129323881741e-05, "clip_ratio/region_mean": 0.0007300336728803813, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 2021.0, "completions/mean_length": 984.86669921875, "completions/mean_terminated_length": 888.2181396484375, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.15707450111707053, "epoch": 0.08892128279883382, "frac_reward_zero_std": 0.0, "grad_norm": 0.011188686825335026, "learning_rate": 2e-05, "loss": 0.03420940414071083, "num_tokens": 3332205.0, "reward": 0.0857747420668602, "reward_std": 0.5992581844329834, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971747398376465, "rewards/length_penalty/mean": -0.4808919131755829, "rewards/length_penalty/std": 0.2573603391647339, "sampling/importance_sampling_ratio/max": 1.447928786277771, "sampling/importance_sampling_ratio/mean": 0.9946839809417725, "sampling/importance_sampling_ratio/min": 0.5939050912857056, "sampling/sampling_logp_difference/max": 0.5210357904434204, "sampling/sampling_logp_difference/mean": 0.010414060205221176, "step": 61, "step_time": 26.998896970879287 }, { "clip_ratio/high_max": 0.0005939534506372487, "clip_ratio/high_mean": 0.0005939534506372487, "clip_ratio/low_mean": 0.00015263191501920423, "clip_ratio/low_min": 0.00015263191501920423, "clip_ratio/region_mean": 0.0007465853608058145, "completions/clipped_ratio": 0.25, "completions/max_length": 2048.0, "completions/max_terminated_length": 2044.0, "completions/mean_length": 1097.7667236328125, "completions/mean_terminated_length": 781.022216796875, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.333564189573129, "epoch": 0.09037900874635568, "frac_reward_zero_std": 0.0, "grad_norm": 0.015973957255482674, "learning_rate": 2e-05, "loss": 0.10216701030731201, "num_tokens": 3402511.0, "reward": -0.30268555879592896, "reward_std": 0.5816874504089355, "rewards/correctness/mean": 0.23333333432674408, "rewards/correctness/std": 0.42652183771133423, "rewards/length_penalty/mean": -0.5360189080238342, "rewards/length_penalty/std": 0.36388248205184937, "sampling/importance_sampling_ratio/max": 1.866272211074829, "sampling/importance_sampling_ratio/mean": 0.988468587398529, "sampling/importance_sampling_ratio/min": 0.521647572517395, "sampling/sampling_logp_difference/max": 0.6507630348205566, "sampling/sampling_logp_difference/mean": 0.019739948213100433, "step": 62, "step_time": 27.611096700653434 }, { "clip_ratio/high_max": 0.001043901705997996, "clip_ratio/high_mean": 0.001043901705997996, "clip_ratio/low_mean": 0.00015817767052794807, "clip_ratio/low_min": 0.00015817767052794807, "clip_ratio/region_mean": 0.001202079370462646, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1840.0, "completions/mean_length": 949.9500732421875, "completions/mean_terminated_length": 756.176513671875, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.3158053010702133, "epoch": 0.09183673469387756, "frac_reward_zero_std": 0.0, "grad_norm": 0.010108426213264465, "learning_rate": 2e-05, "loss": 0.035676177591085434, "num_tokens": 3464468.0, "reward": 0.16949057579040527, "reward_std": 0.6702336072921753, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.4638427793979645, "rewards/length_penalty/std": 0.27556726336479187, "sampling/importance_sampling_ratio/max": 1.4403526782989502, "sampling/importance_sampling_ratio/mean": 0.9890905022621155, "sampling/importance_sampling_ratio/min": 0.5818617343902588, "sampling/sampling_logp_difference/max": 0.5415223836898804, "sampling/sampling_logp_difference/mean": 0.018299730494618416, "step": 63, "step_time": 26.40689355507493 }, { "clip_ratio/high_max": 0.0007529740202395866, "clip_ratio/high_mean": 0.0007529740202395866, "clip_ratio/low_mean": 0.00015265539210910598, "clip_ratio/low_min": 0.00015265539210910598, "clip_ratio/region_mean": 0.0009056293977967774, "completions/clipped_ratio": 0.18333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 1879.0, "completions/mean_length": 1017.9500732421875, "completions/mean_terminated_length": 786.7142944335938, "completions/min_length": 159.0, "completions/min_terminated_length": 159.0, "entropy": 0.30281171202659607, "epoch": 0.09329446064139942, "frac_reward_zero_std": 0.0, "grad_norm": 0.012335984967648983, "learning_rate": 2e-05, "loss": 0.046371761709451675, "num_tokens": 3530505.0, "reward": -0.04704590141773224, "reward_std": 0.7366029620170593, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.4970459043979645, "rewards/length_penalty/std": 0.3209838271141052, "sampling/importance_sampling_ratio/max": 1.4508121013641357, "sampling/importance_sampling_ratio/mean": 0.9895980954170227, "sampling/importance_sampling_ratio/min": 0.1913512945175171, "sampling/sampling_logp_difference/max": 1.653644323348999, "sampling/sampling_logp_difference/mean": 0.017647288739681244, "step": 64, "step_time": 27.50379885500297 }, { "clip_ratio/high_max": 0.0010249692228777956, "clip_ratio/high_mean": 0.0010249692228777956, "clip_ratio/low_mean": 6.801817289669998e-05, "clip_ratio/low_min": 6.801817289669998e-05, "clip_ratio/region_mean": 0.0010929873872858782, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1941.0, "completions/mean_length": 745.1333618164062, "completions/mean_terminated_length": 700.2069091796875, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.23717784136533737, "epoch": 0.09475218658892129, "frac_reward_zero_std": 0.0, "grad_norm": 0.013865753076970577, "learning_rate": 2e-05, "loss": 0.03723486512899399, "num_tokens": 3579353.0, "reward": 0.43616539239883423, "reward_std": 0.44210493564605713, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.3638346493244171, "rewards/length_penalty/std": 0.2601844072341919, "sampling/importance_sampling_ratio/max": 1.4597342014312744, "sampling/importance_sampling_ratio/mean": 0.9913047552108765, "sampling/importance_sampling_ratio/min": 0.18348518013954163, "sampling/sampling_logp_difference/max": 1.695621371269226, "sampling/sampling_logp_difference/mean": 0.015431130304932594, "step": 65, "step_time": 25.39121359703131 }, { "clip_ratio/high_max": 0.0008574471285101026, "clip_ratio/high_mean": 0.0008574471285101026, "clip_ratio/low_mean": 1.902587488681699e-05, "clip_ratio/low_min": 1.902587488681699e-05, "clip_ratio/region_mean": 0.0008764730009716004, "completions/clipped_ratio": 0.0, "completions/max_length": 1282.0, "completions/max_terminated_length": 1282.0, "completions/mean_length": 842.800048828125, "completions/mean_terminated_length": 842.800048828125, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.16640034566322962, "epoch": 0.09620991253644315, "frac_reward_zero_std": 0.0, "grad_norm": 0.01003518607467413, "learning_rate": 2e-05, "loss": -0.012614110484719276, "num_tokens": 3636131.0, "reward": 0.30514323711395264, "reward_std": 0.4085974097251892, "rewards/correctness/mean": 0.7166666388511658, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.4115234315395355, "rewards/length_penalty/std": 0.11401911079883575, "sampling/importance_sampling_ratio/max": 1.459780216217041, "sampling/importance_sampling_ratio/mean": 0.9943774938583374, "sampling/importance_sampling_ratio/min": 0.6570032238960266, "sampling/sampling_logp_difference/max": 0.42006635665893555, "sampling/sampling_logp_difference/mean": 0.01115433406084776, "step": 66, "step_time": 17.655639443080872 }, { "clip_ratio/high_max": 0.0009674198275509601, "clip_ratio/high_mean": 0.0009674198275509601, "clip_ratio/low_mean": 0.0002249472114878396, "clip_ratio/low_min": 0.0002249472114878396, "clip_ratio/region_mean": 0.001192367043889438, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 2048.0, "completions/max_terminated_length": 2024.0, "completions/mean_length": 1367.0667724609375, "completions/mean_terminated_length": 1075.2381591796875, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "entropy": 0.3390492995580037, "epoch": 0.09766763848396501, "frac_reward_zero_std": 0.0, "grad_norm": 0.015688007697463036, "learning_rate": 2e-05, "loss": 0.12193520367145538, "num_tokens": 3723895.0, "reward": -0.23417970538139343, "reward_std": 0.7415551543235779, "rewards/correctness/mean": 0.4333333373069763, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.6675130128860474, "rewards/length_penalty/std": 0.2935273051261902, "sampling/importance_sampling_ratio/max": 1.76029372215271, "sampling/importance_sampling_ratio/mean": 0.9881556034088135, "sampling/importance_sampling_ratio/min": 0.553299069404602, "sampling/sampling_logp_difference/max": 0.5918565988540649, "sampling/sampling_logp_difference/mean": 0.02006438374519348, "step": 67, "step_time": 29.594953797059134 }, { "clip_ratio/high_max": 0.0010029586652914684, "clip_ratio/high_mean": 0.0010029586652914684, "clip_ratio/low_mean": 6.584194488823414e-05, "clip_ratio/low_min": 6.584194488823414e-05, "clip_ratio/region_mean": 0.0010688006101797025, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 2028.0, "completions/mean_length": 731.1333618164062, "completions/mean_terminated_length": 685.72412109375, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.234903650979201, "epoch": 0.09912536443148688, "frac_reward_zero_std": 0.0, "grad_norm": 0.016613559797406197, "learning_rate": 2e-05, "loss": -0.02735317125916481, "num_tokens": 3772593.0, "reward": 0.37633466720581055, "reward_std": 0.5642049908638, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.3569987118244171, "rewards/length_penalty/std": 0.23382732272148132, "sampling/importance_sampling_ratio/max": 1.4509068727493286, "sampling/importance_sampling_ratio/mean": 0.991604745388031, "sampling/importance_sampling_ratio/min": 0.6131345629692078, "sampling/sampling_logp_difference/max": 0.4891709089279175, "sampling/sampling_logp_difference/mean": 0.014993232674896717, "step": 68, "step_time": 25.669897325802594 }, { "clip_ratio/high_max": 0.0006246660535301393, "clip_ratio/high_mean": 0.0006246660535301393, "clip_ratio/low_mean": 4.8398287617601454e-05, "clip_ratio/low_min": 4.8398287617601454e-05, "clip_ratio/region_mean": 0.0006730643459983791, "completions/clipped_ratio": 0.0, "completions/max_length": 1366.0, "completions/max_terminated_length": 1366.0, "completions/mean_length": 694.7667236328125, "completions/mean_terminated_length": 694.7667236328125, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.13991509626309076, "epoch": 0.10058309037900874, "frac_reward_zero_std": 0.0, "grad_norm": 0.009719469584524632, "learning_rate": 2e-05, "loss": -0.004255794454365969, "num_tokens": 3817919.0, "reward": 0.2774251401424408, "reward_std": 0.5339546203613281, "rewards/correctness/mean": 0.6166666746139526, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.33924153447151184, "rewards/length_penalty/std": 0.12143746763467789, "sampling/importance_sampling_ratio/max": 1.378197193145752, "sampling/importance_sampling_ratio/mean": 0.9951502680778503, "sampling/importance_sampling_ratio/min": 0.6503708958625793, "sampling/sampling_logp_difference/max": 0.43021249771118164, "sampling/sampling_logp_difference/mean": 0.009477663785219193, "step": 69, "step_time": 17.807427857769653 }, { "clip_ratio/high_max": 0.0006121192127466202, "clip_ratio/high_mean": 0.0006121192127466202, "clip_ratio/low_mean": 0.00014732059329010858, "clip_ratio/low_min": 0.00014732059329010858, "clip_ratio/region_mean": 0.0007594397854215155, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1808.0, "completions/mean_length": 885.0167236328125, "completions/mean_terminated_length": 844.913818359375, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.23627175887425741, "epoch": 0.10204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.012149685062468052, "learning_rate": 2e-05, "loss": 0.02499573305249214, "num_tokens": 3875290.0, "reward": 0.15119630098342896, "reward_std": 0.5673785209655762, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.43213704228401184, "rewards/length_penalty/std": 0.19533012807369232, "sampling/importance_sampling_ratio/max": 1.4544672966003418, "sampling/importance_sampling_ratio/mean": 0.9921490550041199, "sampling/importance_sampling_ratio/min": 0.6486742496490479, "sampling/sampling_logp_difference/max": 0.43282461166381836, "sampling/sampling_logp_difference/mean": 0.014074113219976425, "step": 70, "step_time": 27.213721254840493 }, { "clip_ratio/high_max": 0.0005193926529803624, "clip_ratio/high_mean": 0.0005193926529803624, "clip_ratio/low_mean": 7.590001526599129e-05, "clip_ratio/low_min": 7.590001526599129e-05, "clip_ratio/region_mean": 0.0005952926730969921, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1921.0, "completions/mean_length": 849.6333618164062, "completions/mean_terminated_length": 808.3103637695312, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.17722671975692114, "epoch": 0.10349854227405247, "frac_reward_zero_std": 0.0, "grad_norm": 0.009361296892166138, "learning_rate": 2e-05, "loss": 0.00986897200345993, "num_tokens": 3930718.0, "reward": 0.15180665254592896, "reward_std": 0.6519619822502136, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.4148600399494171, "rewards/length_penalty/std": 0.2246708869934082, "sampling/importance_sampling_ratio/max": 1.3928110599517822, "sampling/importance_sampling_ratio/mean": 0.9938569068908691, "sampling/importance_sampling_ratio/min": 0.6491281390190125, "sampling/sampling_logp_difference/max": 0.4321250915527344, "sampling/sampling_logp_difference/mean": 0.01106953714042902, "step": 71, "step_time": 25.902137389639392 }, { "clip_ratio/high_max": 0.0010704514182483156, "clip_ratio/high_mean": 0.0010704514182483156, "clip_ratio/low_mean": 8.437880266380186e-05, "clip_ratio/low_min": 8.437880266380186e-05, "clip_ratio/region_mean": 0.001154830203934883, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 2033.0, "completions/mean_length": 880.050048828125, "completions/mean_terminated_length": 725.79248046875, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.27359068890412647, "epoch": 0.10495626822157435, "frac_reward_zero_std": 0.0, "grad_norm": 0.016086820513010025, "learning_rate": 2e-05, "loss": 0.003041524440050125, "num_tokens": 3988201.0, "reward": 0.2702881097793579, "reward_std": 0.606598973274231, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.4297119081020355, "rewards/length_penalty/std": 0.27651119232177734, "sampling/importance_sampling_ratio/max": 1.7209787368774414, "sampling/importance_sampling_ratio/mean": 0.9903214573860168, "sampling/importance_sampling_ratio/min": 0.5492734909057617, "sampling/sampling_logp_difference/max": 0.599158763885498, "sampling/sampling_logp_difference/mean": 0.017103716731071472, "step": 72, "step_time": 26.191329712746665 }, { "clip_ratio/high_max": 0.0009247674946285164, "clip_ratio/high_mean": 0.0009247674946285164, "clip_ratio/low_mean": 7.573514206645389e-05, "clip_ratio/low_min": 7.573514206645389e-05, "clip_ratio/region_mean": 0.0010005026415456086, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1745.0, "completions/mean_length": 751.433349609375, "completions/mean_terminated_length": 729.4576416015625, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.30501004060109455, "epoch": 0.10641399416909621, "frac_reward_zero_std": 0.0, "grad_norm": 0.010819511488080025, "learning_rate": 2e-05, "loss": -0.01672491244971752, "num_tokens": 4037637.0, "reward": 0.36642253398895264, "reward_std": 0.5407053828239441, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.36691081523895264, "rewards/length_penalty/std": 0.2128191888332367, "sampling/importance_sampling_ratio/max": 1.4509228467941284, "sampling/importance_sampling_ratio/mean": 0.989236056804657, "sampling/importance_sampling_ratio/min": 0.6481517553329468, "sampling/sampling_logp_difference/max": 0.43363046646118164, "sampling/sampling_logp_difference/mean": 0.018513785675168037, "step": 73, "step_time": 25.738604004960507 }, { "clip_ratio/high_max": 0.000775772636795106, "clip_ratio/high_mean": 0.000775772636795106, "clip_ratio/low_mean": 0.0001139961532317102, "clip_ratio/low_min": 0.0001139961532317102, "clip_ratio/region_mean": 0.0008897687827508586, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1936.0, "completions/mean_length": 1033.61669921875, "completions/mean_terminated_length": 961.1607666015625, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.2437108432253202, "epoch": 0.10787172011661808, "frac_reward_zero_std": 0.0, "grad_norm": 0.012200290337204933, "learning_rate": 2e-05, "loss": -0.007034962996840477, "num_tokens": 4104524.0, "reward": 0.07863769680261612, "reward_std": 0.6565274596214294, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.5046956539154053, "rewards/length_penalty/std": 0.2536833882331848, "sampling/importance_sampling_ratio/max": 2.0870227813720703, "sampling/importance_sampling_ratio/mean": 0.9919592142105103, "sampling/importance_sampling_ratio/min": 0.5414741039276123, "sampling/sampling_logp_difference/max": 0.7357385158538818, "sampling/sampling_logp_difference/mean": 0.014488711953163147, "step": 74, "step_time": 28.6800376502797 }, { "clip_ratio/high_max": 0.0008456006908090785, "clip_ratio/high_mean": 0.0008456006908090785, "clip_ratio/low_mean": 0.0001688671254669316, "clip_ratio/low_min": 0.0001688671254669316, "clip_ratio/region_mean": 0.0010144678429545213, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1743.0, "completions/mean_length": 875.3167114257812, "completions/mean_terminated_length": 668.37255859375, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.29018878440062207, "epoch": 0.10932944606413994, "frac_reward_zero_std": 0.0, "grad_norm": 0.010037309490144253, "learning_rate": 2e-05, "loss": 0.08295391499996185, "num_tokens": 4162243.0, "reward": 0.23926596343517303, "reward_std": 0.6652383804321289, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.42740070819854736, "rewards/length_penalty/std": 0.29186713695526123, "sampling/importance_sampling_ratio/max": 1.4001084566116333, "sampling/importance_sampling_ratio/mean": 0.9903859496116638, "sampling/importance_sampling_ratio/min": 0.6442928910255432, "sampling/sampling_logp_difference/max": 0.4396018981933594, "sampling/sampling_logp_difference/mean": 0.017068399116396904, "step": 75, "step_time": 26.439316188218072 }, { "clip_ratio/high_max": 0.0007683581789024174, "clip_ratio/high_mean": 0.0007683581789024174, "clip_ratio/low_mean": 8.965843395950894e-05, "clip_ratio/low_min": 8.965843395950894e-05, "clip_ratio/region_mean": 0.0008580166225632032, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1825.0, "completions/mean_length": 862.7667236328125, "completions/mean_terminated_length": 706.2264404296875, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.32068585356076557, "epoch": 0.11078717201166181, "frac_reward_zero_std": 0.0, "grad_norm": 0.013561427593231201, "learning_rate": 2e-05, "loss": 0.03460405021905899, "num_tokens": 4218769.0, "reward": 0.09539388865232468, "reward_std": 0.6922449469566345, "rewards/correctness/mean": 0.5166666507720947, "rewards/correctness/std": 0.5039393305778503, "rewards/length_penalty/mean": -0.42127278447151184, "rewards/length_penalty/std": 0.2805083990097046, "sampling/importance_sampling_ratio/max": 1.427881121635437, "sampling/importance_sampling_ratio/mean": 0.9886918067932129, "sampling/importance_sampling_ratio/min": 0.6494015455245972, "sampling/sampling_logp_difference/max": 0.431704044342041, "sampling/sampling_logp_difference/mean": 0.018998652696609497, "step": 76, "step_time": 25.95738762873225 }, { "clip_ratio/high_max": 0.000573172643877721, "clip_ratio/high_mean": 0.000573172643877721, "clip_ratio/low_mean": 0.0001271530636586249, "clip_ratio/low_min": 0.0001271530636586249, "clip_ratio/region_mean": 0.0007003257002603883, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1871.0, "completions/mean_length": 827.6666870117188, "completions/mean_terminated_length": 612.313720703125, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.32657332470019657, "epoch": 0.11224489795918367, "frac_reward_zero_std": 0.0, "grad_norm": 0.012244322337210178, "learning_rate": 2e-05, "loss": 0.042617566883563995, "num_tokens": 4274079.0, "reward": -0.02080078236758709, "reward_std": 0.6631997227668762, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014302253723, "rewards/length_penalty/mean": -0.4041341245174408, "rewards/length_penalty/std": 0.2911378741264343, "sampling/importance_sampling_ratio/max": 1.7601795196533203, "sampling/importance_sampling_ratio/mean": 0.988547682762146, "sampling/importance_sampling_ratio/min": 0.6220017671585083, "sampling/sampling_logp_difference/max": 0.5654158592224121, "sampling/sampling_logp_difference/mean": 0.019659103825688362, "step": 77, "step_time": 26.511172500206158 }, { "clip_ratio/high_max": 0.0006040186854079366, "clip_ratio/high_mean": 0.0006040186854079366, "clip_ratio/low_mean": 0.00011882732602922867, "clip_ratio/low_min": 0.00011882732602922867, "clip_ratio/region_mean": 0.0007228460065865269, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 2011.0, "completions/mean_length": 1037.86669921875, "completions/mean_terminated_length": 904.4528198242188, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "entropy": 0.22266371299823126, "epoch": 0.11370262390670553, "frac_reward_zero_std": 0.0, "grad_norm": 0.0142426248639822, "learning_rate": 2e-05, "loss": 0.01979541964828968, "num_tokens": 4342531.0, "reward": -0.15677084028720856, "reward_std": 0.6428810358047485, "rewards/correctness/mean": 0.3499999940395355, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.5067708492279053, "rewards/length_penalty/std": 0.26104095578193665, "sampling/importance_sampling_ratio/max": 1.7160686254501343, "sampling/importance_sampling_ratio/mean": 0.9918795228004456, "sampling/importance_sampling_ratio/min": 0.6340979933738708, "sampling/sampling_logp_difference/max": 0.5400359630584717, "sampling/sampling_logp_difference/mean": 0.01398763433098793, "step": 78, "step_time": 27.06211461359635 }, { "clip_ratio/high_max": 0.0004470327755067653, "clip_ratio/high_mean": 0.0004470327755067653, "clip_ratio/low_mean": 0.00013647254066502987, "clip_ratio/low_min": 0.00013647254066502987, "clip_ratio/region_mean": 0.0005835053161717951, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 1006.3333740234375, "completions/mean_terminated_length": 822.5098266601562, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.20668349415063858, "epoch": 0.1151603498542274, "frac_reward_zero_std": 0.0, "grad_norm": 0.009284527972340584, "learning_rate": 2e-05, "loss": 0.06438212096691132, "num_tokens": 4409821.0, "reward": 0.1752929836511612, "reward_std": 0.7006873488426208, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.4913736879825592, "rewards/length_penalty/std": 0.26470717787742615, "sampling/importance_sampling_ratio/max": 1.4403252601623535, "sampling/importance_sampling_ratio/mean": 0.9931297898292542, "sampling/importance_sampling_ratio/min": 0.6608903408050537, "sampling/sampling_logp_difference/max": 0.4141674041748047, "sampling/sampling_logp_difference/mean": 0.012824874371290207, "step": 79, "step_time": 27.364660371094942 }, { "clip_ratio/high_max": 0.000855635866173543, "clip_ratio/high_mean": 0.000855635866173543, "clip_ratio/low_mean": 0.00013942622414712483, "clip_ratio/low_min": 0.00013942622414712483, "clip_ratio/region_mean": 0.0009950621169991791, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 777.9000244140625, "completions/mean_terminated_length": 687.1785888671875, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.2723602006832759, "epoch": 0.11661807580174927, "frac_reward_zero_std": 0.0, "grad_norm": 0.016341570764780045, "learning_rate": 2e-05, "loss": 0.07856422662734985, "num_tokens": 4461305.0, "reward": 0.47016602754592896, "reward_std": 0.568079948425293, "rewards/correctness/mean": 0.8500000238418579, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.37983399629592896, "rewards/length_penalty/std": 0.270346462726593, "sampling/importance_sampling_ratio/max": 1.515631914138794, "sampling/importance_sampling_ratio/mean": 0.990748405456543, "sampling/importance_sampling_ratio/min": 0.6570536494255066, "sampling/sampling_logp_difference/max": 0.41998958587646484, "sampling/sampling_logp_difference/mean": 0.016672134399414062, "step": 80, "step_time": 26.63881313498132 }, { "clip_ratio/high_max": 0.0008328702727643152, "clip_ratio/high_mean": 0.0008328702727643152, "clip_ratio/low_mean": 0.0001242530840196802, "clip_ratio/low_min": 0.0001242530840196802, "clip_ratio/region_mean": 0.0009571233725485703, "completions/clipped_ratio": 0.31666669249534607, "completions/max_length": 2048.0, "completions/max_terminated_length": 2000.0, "completions/mean_length": 959.8167114257812, "completions/mean_terminated_length": 455.53656005859375, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.23871558656295142, "epoch": 0.11807580174927114, "frac_reward_zero_std": 0.0, "grad_norm": 0.013187786564230919, "learning_rate": 2e-05, "loss": 0.06366457045078278, "num_tokens": 4523414.0, "reward": -0.0019938151817768812, "reward_std": 0.8292409181594849, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.46866047382354736, "rewards/length_penalty/std": 0.396953821182251, "sampling/importance_sampling_ratio/max": 1.4598079919815063, "sampling/importance_sampling_ratio/mean": 0.9917196035385132, "sampling/importance_sampling_ratio/min": 0.5851331353187561, "sampling/sampling_logp_difference/max": 0.5359158515930176, "sampling/sampling_logp_difference/mean": 0.014803586527705193, "step": 81, "step_time": 26.944744786946103 }, { "clip_ratio/high_max": 0.0011079669323711034, "clip_ratio/high_mean": 0.0011079669323711034, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011079669323711034, "completions/clipped_ratio": 0.0, "completions/max_length": 922.0, "completions/max_terminated_length": 922.0, "completions/mean_length": 443.0500183105469, "completions/mean_terminated_length": 443.0500183105469, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.19146856168905893, "epoch": 0.119533527696793, "frac_reward_zero_std": 0.0, "grad_norm": 0.0076869698241353035, "learning_rate": 2e-05, "loss": -0.0073247989639639854, "num_tokens": 4553937.0, "reward": 0.43366700410842896, "reward_std": 0.4700399339199066, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.21633300185203552, "rewards/length_penalty/std": 0.08693579584360123, "sampling/importance_sampling_ratio/max": 1.4495785236358643, "sampling/importance_sampling_ratio/mean": 0.9935380816459656, "sampling/importance_sampling_ratio/min": 0.6671428680419922, "sampling/sampling_logp_difference/max": 0.4047510623931885, "sampling/sampling_logp_difference/mean": 0.012338114902377129, "step": 82, "step_time": 11.948872851207852 }, { "clip_ratio/high_max": 0.0008603990378711993, "clip_ratio/high_mean": 0.0008603990378711993, "clip_ratio/low_mean": 0.00012837099590493986, "clip_ratio/low_min": 0.00012837099590493986, "clip_ratio/region_mean": 0.0009887700386267777, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1752.0, "completions/mean_length": 942.3167114257812, "completions/mean_terminated_length": 665.8958740234375, "completions/min_length": 433.0, "completions/min_terminated_length": 433.0, "entropy": 0.3316035866737366, "epoch": 0.12099125364431487, "frac_reward_zero_std": 0.0, "grad_norm": 0.012608137913048267, "learning_rate": 2e-05, "loss": 0.054517023265361786, "num_tokens": 4614906.0, "reward": 0.13988444209098816, "reward_std": 0.6998077630996704, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.46011555194854736, "rewards/length_penalty/std": 0.2946597635746002, "sampling/importance_sampling_ratio/max": 1.5668257474899292, "sampling/importance_sampling_ratio/mean": 0.9886726140975952, "sampling/importance_sampling_ratio/min": 0.6418333649635315, "sampling/sampling_logp_difference/max": 0.4490518569946289, "sampling/sampling_logp_difference/mean": 0.019341254606842995, "step": 83, "step_time": 26.819837241899222 }, { "clip_ratio/high_max": 0.0005293477273274524, "clip_ratio/high_mean": 0.0005293477273274524, "clip_ratio/low_mean": 9.641703218221664e-05, "clip_ratio/low_min": 9.641703218221664e-05, "clip_ratio/region_mean": 0.000625764759509669, "completions/clipped_ratio": 0.0, "completions/max_length": 1947.0, "completions/max_terminated_length": 1947.0, "completions/mean_length": 796.6000366210938, "completions/mean_terminated_length": 796.6000366210938, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.20828798661629358, "epoch": 0.12244897959183673, "frac_reward_zero_std": 0.0, "grad_norm": 0.013983390294015408, "learning_rate": 2e-05, "loss": -0.004879172891378403, "num_tokens": 4667622.0, "reward": 0.19436849653720856, "reward_std": 0.5332536697387695, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.38896483182907104, "rewards/length_penalty/std": 0.25310996174812317, "sampling/importance_sampling_ratio/max": 1.6624352931976318, "sampling/importance_sampling_ratio/mean": 0.9927626252174377, "sampling/importance_sampling_ratio/min": 0.6399871706962585, "sampling/sampling_logp_difference/max": 0.5082836151123047, "sampling/sampling_logp_difference/mean": 0.012836500070989132, "step": 84, "step_time": 24.730451047653332 }, { "clip_ratio/high_max": 0.00036194647691445425, "clip_ratio/high_mean": 0.00036194647691445425, "clip_ratio/low_mean": 7.499000639654696e-05, "clip_ratio/low_min": 7.499000639654696e-05, "clip_ratio/region_mean": 0.0004369364833110012, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1934.0, "completions/mean_length": 654.2000122070312, "completions/mean_terminated_length": 470.11322021484375, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.2506712128718694, "epoch": 0.1239067055393586, "frac_reward_zero_std": 0.0, "grad_norm": 0.01700645312666893, "learning_rate": 2e-05, "loss": 0.0787142887711525, "num_tokens": 4710704.0, "reward": 0.4972330927848816, "reward_std": 0.6686694025993347, "rewards/correctness/mean": 0.8166666626930237, "rewards/correctness/std": 0.39020493626594543, "rewards/length_penalty/mean": -0.3194335997104645, "rewards/length_penalty/std": 0.293500691652298, "sampling/importance_sampling_ratio/max": 1.4508010149002075, "sampling/importance_sampling_ratio/mean": 0.991134524345398, "sampling/importance_sampling_ratio/min": 0.5592718720436096, "sampling/sampling_logp_difference/max": 0.5811195373535156, "sampling/sampling_logp_difference/mean": 0.015687232837080956, "step": 85, "step_time": 24.8261538639199 }, { "clip_ratio/high_max": 0.0007935117804057276, "clip_ratio/high_mean": 0.0007935117804057276, "clip_ratio/low_mean": 0.0002459599963913206, "clip_ratio/low_min": 0.0002459599963913206, "clip_ratio/region_mean": 0.0010394717889236442, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1464.0, "completions/mean_length": 867.0333862304688, "completions/mean_terminated_length": 630.8399658203125, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.2866263687610626, "epoch": 0.12536443148688048, "frac_reward_zero_std": 0.0, "grad_norm": 0.01707608811557293, "learning_rate": 2e-05, "loss": 0.08202922344207764, "num_tokens": 4767436.0, "reward": 0.20997722446918488, "reward_std": 0.6878211498260498, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.4233561158180237, "rewards/length_penalty/std": 0.2928031086921692, "sampling/importance_sampling_ratio/max": 1.4199771881103516, "sampling/importance_sampling_ratio/mean": 0.9901440143585205, "sampling/importance_sampling_ratio/min": 0.2758060097694397, "sampling/sampling_logp_difference/max": 1.288057565689087, "sampling/sampling_logp_difference/mean": 0.01677057519555092, "step": 86, "step_time": 26.44699452095665 }, { "clip_ratio/high_max": 0.000700932631540733, "clip_ratio/high_mean": 0.000700932631540733, "clip_ratio/low_mean": 0.0001029938454545724, "clip_ratio/low_min": 0.0001029938454545724, "clip_ratio/region_mean": 0.0008039264551674327, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 2039.0, "completions/mean_length": 953.550048828125, "completions/mean_terminated_length": 854.0545043945312, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.21370024979114532, "epoch": 0.12682215743440234, "frac_reward_zero_std": 0.0, "grad_norm": 0.008855806663632393, "learning_rate": 2e-05, "loss": -0.005549025721848011, "num_tokens": 4829639.0, "reward": 0.26773276925086975, "reward_std": 0.6256986260414124, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.4656005799770355, "rewards/length_penalty/std": 0.24547874927520752, "sampling/importance_sampling_ratio/max": 1.4311411380767822, "sampling/importance_sampling_ratio/mean": 0.992712140083313, "sampling/importance_sampling_ratio/min": 0.6615297198295593, "sampling/sampling_logp_difference/max": 0.41320037841796875, "sampling/sampling_logp_difference/mean": 0.013411559164524078, "step": 87, "step_time": 26.37262093811296 }, { "clip_ratio/high_max": 0.0004775548926166569, "clip_ratio/high_mean": 0.0004775548926166569, "clip_ratio/low_mean": 9.304770234545383e-05, "clip_ratio/low_min": 9.304770234545383e-05, "clip_ratio/region_mean": 0.0005706025925367916, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1775.0, "completions/mean_length": 971.1834106445312, "completions/mean_terminated_length": 701.9791870117188, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.3344749758640925, "epoch": 0.1282798833819242, "frac_reward_zero_std": 0.0, "grad_norm": 0.010389057919383049, "learning_rate": 2e-05, "loss": 0.000678627286106348, "num_tokens": 4895050.0, "reward": -0.05754394829273224, "reward_std": 0.6811898350715637, "rewards/correctness/mean": 0.4166666567325592, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.47421061992645264, "rewards/length_penalty/std": 0.29730790853500366, "sampling/importance_sampling_ratio/max": 2.342852830886841, "sampling/importance_sampling_ratio/mean": 0.98845374584198, "sampling/importance_sampling_ratio/min": 0.6082951426506042, "sampling/sampling_logp_difference/max": 0.8513693809509277, "sampling/sampling_logp_difference/mean": 0.019974932074546814, "step": 88, "step_time": 26.829658325761557 }, { "clip_ratio/high_max": 0.0011431698706777145, "clip_ratio/high_mean": 0.0011431698706777145, "clip_ratio/low_mean": 8.953552848349015e-05, "clip_ratio/low_min": 8.953552848349015e-05, "clip_ratio/region_mean": 0.0012327053894599278, "completions/clipped_ratio": 0.0, "completions/max_length": 1462.0, "completions/max_terminated_length": 1462.0, "completions/mean_length": 723.7833862304688, "completions/mean_terminated_length": 723.7833862304688, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.20457501461108527, "epoch": 0.12973760932944606, "frac_reward_zero_std": 0.0, "grad_norm": 0.00969801563769579, "learning_rate": 2e-05, "loss": -0.008164532482624054, "num_tokens": 4943227.0, "reward": 0.513256847858429, "reward_std": 0.3979385793209076, "rewards/correctness/mean": 0.8666666746139526, "rewards/correctness/std": 0.34280332922935486, "rewards/length_penalty/mean": -0.3534098267555237, "rewards/length_penalty/std": 0.15708020329475403, "sampling/importance_sampling_ratio/max": 1.7208794355392456, "sampling/importance_sampling_ratio/mean": 0.9932364821434021, "sampling/importance_sampling_ratio/min": 0.5768768191337585, "sampling/sampling_logp_difference/max": 0.5501265525817871, "sampling/sampling_logp_difference/mean": 0.012802796438336372, "step": 89, "step_time": 19.591808903263882 }, { "clip_ratio/high_max": 0.001249313597024108, "clip_ratio/high_mean": 0.001249313597024108, "clip_ratio/low_mean": 0.00016507423909691474, "clip_ratio/low_min": 0.00016507423909691474, "clip_ratio/region_mean": 0.0014143877973159154, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1938.0, "completions/mean_length": 758.7500610351562, "completions/mean_terminated_length": 690.8947143554688, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.3826302985350291, "epoch": 0.13119533527696792, "frac_reward_zero_std": 0.0, "grad_norm": 0.01774374209344387, "learning_rate": 2e-05, "loss": 0.0025574974715709686, "num_tokens": 4994272.0, "reward": 0.4628499448299408, "reward_std": 0.5554989576339722, "rewards/correctness/mean": 0.8333333134651184, "rewards/correctness/std": 0.3758230209350586, "rewards/length_penalty/mean": -0.3704833984375, "rewards/length_penalty/std": 0.24367263913154602, "sampling/importance_sampling_ratio/max": 1.4403685331344604, "sampling/importance_sampling_ratio/mean": 0.9864087104797363, "sampling/importance_sampling_ratio/min": 0.644669771194458, "sampling/sampling_logp_difference/max": 0.43901705741882324, "sampling/sampling_logp_difference/mean": 0.023107297718524933, "step": 90, "step_time": 26.096014055190608 }, { "clip_ratio/high_max": 0.0011761372346275796, "clip_ratio/high_mean": 0.0011761372346275796, "clip_ratio/low_mean": 0.00015822998830117285, "clip_ratio/low_min": 0.00015822998830117285, "clip_ratio/region_mean": 0.0013343672423313062, "completions/clipped_ratio": 0.0, "completions/max_length": 1663.0, "completions/max_terminated_length": 1663.0, "completions/mean_length": 617.683349609375, "completions/mean_terminated_length": 617.683349609375, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.23400658120711645, "epoch": 0.1326530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.012364847585558891, "learning_rate": 2e-05, "loss": 0.013461099937558174, "num_tokens": 5034703.0, "reward": 0.4983968436717987, "reward_std": 0.4498032331466675, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.30160319805145264, "rewards/length_penalty/std": 0.14153972268104553, "sampling/importance_sampling_ratio/max": 1.3339767456054688, "sampling/importance_sampling_ratio/mean": 0.9915993213653564, "sampling/importance_sampling_ratio/min": 0.6506576538085938, "sampling/sampling_logp_difference/max": 0.42977166175842285, "sampling/sampling_logp_difference/mean": 0.014409150928258896, "step": 91, "step_time": 20.228758974932134 }, { "clip_ratio/high_max": 0.0011599833572593827, "clip_ratio/high_mean": 0.0011599833572593827, "clip_ratio/low_mean": 7.72899159831771e-05, "clip_ratio/low_min": 7.72899159831771e-05, "clip_ratio/region_mean": 0.001237273245351389, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1521.0, "completions/mean_length": 877.933349609375, "completions/mean_terminated_length": 643.9199829101562, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.24022843688726425, "epoch": 0.13411078717201166, "frac_reward_zero_std": 0.0, "grad_norm": 0.009864171966910362, "learning_rate": 2e-05, "loss": 0.018639594316482544, "num_tokens": 5092429.0, "reward": 0.3713216483592987, "reward_std": 0.6548762917518616, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.42867839336395264, "rewards/length_penalty/std": 0.28254130482673645, "sampling/importance_sampling_ratio/max": 1.8417235612869263, "sampling/importance_sampling_ratio/mean": 0.9913477301597595, "sampling/importance_sampling_ratio/min": 0.5614640116691589, "sampling/sampling_logp_difference/max": 0.6107019186019897, "sampling/sampling_logp_difference/mean": 0.014954106882214546, "step": 92, "step_time": 26.760359250940382 }, { "clip_ratio/high_max": 0.0007670078193768859, "clip_ratio/high_mean": 0.0007670078193768859, "clip_ratio/low_mean": 6.530691219571357e-05, "clip_ratio/low_min": 6.530691219571357e-05, "clip_ratio/region_mean": 0.0008323147339979187, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1655.0, "completions/mean_length": 649.5000610351562, "completions/mean_terminated_length": 625.796630859375, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.18720209846893945, "epoch": 0.13556851311953352, "frac_reward_zero_std": 0.0, "grad_norm": 0.012682660482823849, "learning_rate": 2e-05, "loss": 0.010452114045619965, "num_tokens": 5135879.0, "reward": 0.23286134004592896, "reward_std": 0.5095825791358948, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.317138671875, "rewards/length_penalty/std": 0.184276282787323, "sampling/importance_sampling_ratio/max": 1.467110276222229, "sampling/importance_sampling_ratio/mean": 0.9933242201805115, "sampling/importance_sampling_ratio/min": 0.6245324015617371, "sampling/sampling_logp_difference/max": 0.4707520008087158, "sampling/sampling_logp_difference/mean": 0.012335265055298805, "step": 93, "step_time": 24.67962512979284 }, { "clip_ratio/high_max": 0.0006980508672616755, "clip_ratio/high_mean": 0.0006980508672616755, "clip_ratio/low_mean": 0.00016362378179716566, "clip_ratio/low_min": 0.00016362378179716566, "clip_ratio/region_mean": 0.0008616746442082027, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1909.0, "completions/mean_length": 895.050048828125, "completions/mean_terminated_length": 717.673095703125, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.3231743698318799, "epoch": 0.13702623906705538, "frac_reward_zero_std": 0.0, "grad_norm": 0.011201799847185612, "learning_rate": 2e-05, "loss": 0.032010000199079514, "num_tokens": 5194552.0, "reward": 0.19629721343517303, "reward_std": 0.6891112327575684, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.4370361268520355, "rewards/length_penalty/std": 0.2695411741733551, "sampling/importance_sampling_ratio/max": 1.4182049036026, "sampling/importance_sampling_ratio/mean": 0.9891645312309265, "sampling/importance_sampling_ratio/min": 0.6431248188018799, "sampling/sampling_logp_difference/max": 0.44141650199890137, "sampling/sampling_logp_difference/mean": 0.018724149093031883, "step": 94, "step_time": 26.034434655681252 }, { "clip_ratio/high_max": 0.0008400467486353591, "clip_ratio/high_mean": 0.0008400467486353591, "clip_ratio/low_mean": 9.523039382960026e-05, "clip_ratio/low_min": 9.523039382960026e-05, "clip_ratio/region_mean": 0.0009352771448902786, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1336.0, "completions/mean_length": 526.933349609375, "completions/mean_terminated_length": 501.1525573730469, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.24195876717567444, "epoch": 0.13848396501457727, "frac_reward_zero_std": 0.0, "grad_norm": 0.012131198309361935, "learning_rate": 2e-05, "loss": 0.00936015136539936, "num_tokens": 5230318.0, "reward": 0.5760416984558105, "reward_std": 0.5255236625671387, "rewards/correctness/mean": 0.8333333134651184, "rewards/correctness/std": 0.3758230209350586, "rewards/length_penalty/mean": -0.25729167461395264, "rewards/length_penalty/std": 0.17985829710960388, "sampling/importance_sampling_ratio/max": 1.3941067457199097, "sampling/importance_sampling_ratio/mean": 0.9913172125816345, "sampling/importance_sampling_ratio/min": 0.6241722106933594, "sampling/sampling_logp_difference/max": 0.4713289737701416, "sampling/sampling_logp_difference/mean": 0.015595714561641216, "step": 95, "step_time": 24.485636703902856 }, { "clip_ratio/high_max": 0.0008672485249311043, "clip_ratio/high_mean": 0.0008672485249311043, "clip_ratio/low_mean": 0.00011490157824785759, "clip_ratio/low_min": 0.00011490157824785759, "clip_ratio/region_mean": 0.0009821500910523657, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1849.0, "completions/mean_length": 695.6666870117188, "completions/mean_terminated_length": 599.0714721679688, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.3335142781337102, "epoch": 0.13994169096209913, "frac_reward_zero_std": 0.0, "grad_norm": 0.010053694248199463, "learning_rate": 2e-05, "loss": 0.007662919815629721, "num_tokens": 5276378.0, "reward": 0.04365234449505806, "reward_std": 0.5776897072792053, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.3396809995174408, "rewards/length_penalty/std": 0.2500426769256592, "sampling/importance_sampling_ratio/max": 1.3981760740280151, "sampling/importance_sampling_ratio/mean": 0.9888298511505127, "sampling/importance_sampling_ratio/min": 0.6447350978851318, "sampling/sampling_logp_difference/max": 0.4389157295227051, "sampling/sampling_logp_difference/mean": 0.019536839798092842, "step": 96, "step_time": 25.759525523055345 }, { "clip_ratio/high_max": 0.0008366378121233234, "clip_ratio/high_mean": 0.0008366378121233234, "clip_ratio/low_mean": 6.1031064736501626e-05, "clip_ratio/low_min": 6.1031064736501626e-05, "clip_ratio/region_mean": 0.0008976688744345059, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 2036.0, "completions/mean_length": 1042.300048828125, "completions/mean_terminated_length": 970.46435546875, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.2069831813375155, "epoch": 0.141399416909621, "frac_reward_zero_std": 0.0, "grad_norm": 0.013124002143740654, "learning_rate": 2e-05, "loss": 0.02402597665786743, "num_tokens": 5342656.0, "reward": 0.3410644829273224, "reward_std": 0.5162246227264404, "rewards/correctness/mean": 0.8500000238418579, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.5089355707168579, "rewards/length_penalty/std": 0.25521671772003174, "sampling/importance_sampling_ratio/max": 1.467110276222229, "sampling/importance_sampling_ratio/mean": 0.9927272796630859, "sampling/importance_sampling_ratio/min": 0.6346305012702942, "sampling/sampling_logp_difference/max": 0.4547123312950134, "sampling/sampling_logp_difference/mean": 0.012942169792950153, "step": 97, "step_time": 26.65434269607067 }, { "clip_ratio/high_max": 0.0005699570174328983, "clip_ratio/high_mean": 0.0005699570174328983, "clip_ratio/low_mean": 2.466577764910956e-05, "clip_ratio/low_min": 2.466577764910956e-05, "clip_ratio/region_mean": 0.0005946227999326462, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1804.0, "completions/mean_length": 646.1000366210938, "completions/mean_terminated_length": 597.7586059570312, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.19671294589837393, "epoch": 0.14285714285714285, "frac_reward_zero_std": 0.0, "grad_norm": 0.011593547649681568, "learning_rate": 2e-05, "loss": 0.03229053318500519, "num_tokens": 5385912.0, "reward": 0.5845215320587158, "reward_std": 0.4700944423675537, "rewards/correctness/mean": 0.8999999761581421, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.31547850370407104, "rewards/length_penalty/std": 0.2142878621816635, "sampling/importance_sampling_ratio/max": 1.5550326108932495, "sampling/importance_sampling_ratio/mean": 0.9931764006614685, "sampling/importance_sampling_ratio/min": 0.6677953004837036, "sampling/sampling_logp_difference/max": 0.44149649143218994, "sampling/sampling_logp_difference/mean": 0.012409945018589497, "step": 98, "step_time": 25.017244498012587 }, { "clip_ratio/high_max": 0.0007077873306116089, "clip_ratio/high_mean": 0.0007077873306116089, "clip_ratio/low_mean": 0.0002881991070656416, "clip_ratio/low_min": 0.0002881991070656416, "clip_ratio/region_mean": 0.0009959864595051233, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1926.0, "completions/mean_length": 1104.300048828125, "completions/mean_terminated_length": 868.375, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.34258048236370087, "epoch": 0.14431486880466474, "frac_reward_zero_std": 0.0, "grad_norm": 0.015076699666678905, "learning_rate": 2e-05, "loss": 0.022467955946922302, "num_tokens": 5456770.0, "reward": 0.02745768427848816, "reward_std": 0.7769045233726501, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.5392090082168579, "rewards/length_penalty/std": 0.3212955892086029, "sampling/importance_sampling_ratio/max": 1.7142083644866943, "sampling/importance_sampling_ratio/mean": 0.9883654117584229, "sampling/importance_sampling_ratio/min": 0.6409561634063721, "sampling/sampling_logp_difference/max": 0.5389513969421387, "sampling/sampling_logp_difference/mean": 0.02026149071753025, "step": 99, "step_time": 28.01622262224555 }, { "clip_ratio/high_max": 0.0006826348374791754, "clip_ratio/high_mean": 0.0006826348374791754, "clip_ratio/low_mean": 0.0002501061159515909, "clip_ratio/low_min": 0.0002501061159515909, "clip_ratio/region_mean": 0.0009327409352408722, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1678.0, "completions/mean_length": 847.1000366210938, "completions/mean_terminated_length": 688.4906005859375, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.3057691554228465, "epoch": 0.1457725947521866, "frac_reward_zero_std": 0.0, "grad_norm": 0.012148505076766014, "learning_rate": 2e-05, "loss": 0.044301412999629974, "num_tokens": 5513056.0, "reward": 0.00304361991584301, "reward_std": 0.6464745998382568, "rewards/correctness/mean": 0.4166666567325592, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.41362303495407104, "rewards/length_penalty/std": 0.26829221844673157, "sampling/importance_sampling_ratio/max": 1.4854363203048706, "sampling/importance_sampling_ratio/mean": 0.9892914295196533, "sampling/importance_sampling_ratio/min": 0.5826400518417358, "sampling/sampling_logp_difference/max": 0.5401856899261475, "sampling/sampling_logp_difference/mean": 0.018160300329327583, "step": 100, "step_time": 25.87883016304113 }, { "clip_ratio/high_max": 0.0008231871761381626, "clip_ratio/high_mean": 0.0008231871761381626, "clip_ratio/low_mean": 0.00010636348937017222, "clip_ratio/low_min": 0.00010636348937017222, "clip_ratio/region_mean": 0.000929550655807058, "completions/clipped_ratio": 0.0, "completions/max_length": 1616.0, "completions/max_terminated_length": 1616.0, "completions/mean_length": 684.6000366210938, "completions/mean_terminated_length": 684.6000366210938, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.21437226235866547, "epoch": 0.14723032069970846, "frac_reward_zero_std": 0.0, "grad_norm": 0.012079442851245403, "learning_rate": 2e-05, "loss": -0.00972786732017994, "num_tokens": 5558272.0, "reward": 0.39905601739883423, "reward_std": 0.47673743963241577, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.33427733182907104, "rewards/length_penalty/std": 0.17992880940437317, "sampling/importance_sampling_ratio/max": 1.4278860092163086, "sampling/importance_sampling_ratio/mean": 0.9925777912139893, "sampling/importance_sampling_ratio/min": 0.652592658996582, "sampling/sampling_logp_difference/max": 0.4268021583557129, "sampling/sampling_logp_difference/mean": 0.013579588383436203, "step": 101, "step_time": 20.694484879029915 }, { "clip_ratio/high_max": 0.000667893347175171, "clip_ratio/high_mean": 0.000667893347175171, "clip_ratio/low_mean": 0.00019019936007680371, "clip_ratio/low_min": 0.00019019936007680371, "clip_ratio/region_mean": 0.0008580927096772939, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1866.0, "completions/mean_length": 1047.416748046875, "completions/mean_terminated_length": 847.2999877929688, "completions/min_length": 456.0, "completions/min_terminated_length": 456.0, "entropy": 0.22797627747058868, "epoch": 0.14868804664723032, "frac_reward_zero_std": 0.0, "grad_norm": 0.015009243972599506, "learning_rate": 2e-05, "loss": 0.043725211173295975, "num_tokens": 5625017.0, "reward": 0.03856608271598816, "reward_std": 0.6842026710510254, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.5114338994026184, "rewards/length_penalty/std": 0.2826176881790161, "sampling/importance_sampling_ratio/max": 1.7557878494262695, "sampling/importance_sampling_ratio/mean": 0.9918825030326843, "sampling/importance_sampling_ratio/min": 0.48450207710266113, "sampling/sampling_logp_difference/max": 0.7246335744857788, "sampling/sampling_logp_difference/mean": 0.014599970541894436, "step": 102, "step_time": 26.755916321650147 }, { "clip_ratio/high_max": 0.0009433443968494734, "clip_ratio/high_mean": 0.0009433443968494734, "clip_ratio/low_mean": 4.761904710903764e-05, "clip_ratio/low_min": 4.761904710903764e-05, "clip_ratio/region_mean": 0.0009909634439585109, "completions/clipped_ratio": 0.0, "completions/max_length": 574.0, "completions/max_terminated_length": 574.0, "completions/mean_length": 353.1000061035156, "completions/mean_terminated_length": 353.1000061035156, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 0.1891030122836431, "epoch": 0.15014577259475217, "frac_reward_zero_std": 0.0, "grad_norm": 0.008301123045384884, "learning_rate": 2e-05, "loss": 0.001749962568283081, "num_tokens": 5649073.0, "reward": 0.7442545890808105, "reward_std": 0.30027633905410767, "rewards/correctness/mean": 0.9166666865348816, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.17241211235523224, "rewards/length_penalty/std": 0.05345404893159866, "sampling/importance_sampling_ratio/max": 1.443211317062378, "sampling/importance_sampling_ratio/mean": 0.9940182566642761, "sampling/importance_sampling_ratio/min": 0.6817989945411682, "sampling/sampling_logp_difference/max": 0.38302040100097656, "sampling/sampling_logp_difference/mean": 0.012686309404671192, "step": 103, "step_time": 8.264996593119577 }, { "clip_ratio/high_max": 0.0017504769057268277, "clip_ratio/high_mean": 0.0017504769057268277, "clip_ratio/low_mean": 0.00015876545148785226, "clip_ratio/low_min": 0.00015876545148785226, "clip_ratio/region_mean": 0.0019092423899564892, "completions/clipped_ratio": 0.21666668355464935, "completions/max_length": 2048.0, "completions/max_terminated_length": 1798.0, "completions/mean_length": 1092.5667724609375, "completions/mean_terminated_length": 828.2978515625, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.2928629020849864, "epoch": 0.15160349854227406, "frac_reward_zero_std": 0.0, "grad_norm": 0.011957842856645584, "learning_rate": 2e-05, "loss": 0.022531531751155853, "num_tokens": 5721567.0, "reward": -0.0668131560087204, "reward_std": 0.6712607145309448, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.5334798097610474, "rewards/length_penalty/std": 0.31666451692581177, "sampling/importance_sampling_ratio/max": 1.4848307371139526, "sampling/importance_sampling_ratio/mean": 0.9893854260444641, "sampling/importance_sampling_ratio/min": 0.43411290645599365, "sampling/sampling_logp_difference/max": 0.8344506025314331, "sampling/sampling_logp_difference/mean": 0.018202045932412148, "step": 104, "step_time": 29.20547517691739 }, { "clip_ratio/high_max": 0.0007943510960709924, "clip_ratio/high_mean": 0.0007943510960709924, "clip_ratio/low_mean": 0.00014659620016270006, "clip_ratio/low_min": 0.00014659620016270006, "clip_ratio/region_mean": 0.0009409472986590117, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1329.0, "completions/mean_length": 641.7500610351562, "completions/mean_terminated_length": 617.915283203125, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.22921649614969888, "epoch": 0.15306122448979592, "frac_reward_zero_std": 0.0, "grad_norm": 0.00924638845026493, "learning_rate": 2e-05, "loss": 0.003329500090330839, "num_tokens": 5767162.0, "reward": 0.16997885704040527, "reward_std": 0.6293733716011047, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.3133544921875, "rewards/length_penalty/std": 0.171682208776474, "sampling/importance_sampling_ratio/max": 1.4909831285476685, "sampling/importance_sampling_ratio/mean": 0.9919578433036804, "sampling/importance_sampling_ratio/min": 0.6197322607040405, "sampling/sampling_logp_difference/max": 0.4784677028656006, "sampling/sampling_logp_difference/mean": 0.014817951247096062, "step": 105, "step_time": 25.627611643401906 }, { "clip_ratio/high_max": 0.001136383895451824, "clip_ratio/high_mean": 0.001136383895451824, "clip_ratio/low_mean": 1.6441419575130567e-05, "clip_ratio/low_min": 1.6441419575130567e-05, "clip_ratio/region_mean": 0.00115282532836621, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1839.0, "completions/mean_length": 922.0167236328125, "completions/mean_terminated_length": 841.5892944335938, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.2763413538535436, "epoch": 0.15451895043731778, "frac_reward_zero_std": 0.0, "grad_norm": 0.015668585896492004, "learning_rate": 2e-05, "loss": 0.04884755611419678, "num_tokens": 5827893.0, "reward": 0.449796587228775, "reward_std": 0.48367294669151306, "rewards/correctness/mean": 0.8999999761581421, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.45020344853401184, "rewards/length_penalty/std": 0.2587897777557373, "sampling/importance_sampling_ratio/max": 1.450933814048767, "sampling/importance_sampling_ratio/mean": 0.990755021572113, "sampling/importance_sampling_ratio/min": 0.6321578025817871, "sampling/sampling_logp_difference/max": 0.4586162567138672, "sampling/sampling_logp_difference/mean": 0.016498787328600883, "step": 106, "step_time": 27.20574454916641 }, { "clip_ratio/high_max": 0.0007279627946748709, "clip_ratio/high_mean": 0.0007279627946748709, "clip_ratio/low_mean": 6.508324440801516e-05, "clip_ratio/low_min": 6.508324440801516e-05, "clip_ratio/region_mean": 0.0007930460415082052, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 504.16668701171875, "completions/mean_terminated_length": 504.16668701171875, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.16326933602492014, "epoch": 0.15597667638483964, "frac_reward_zero_std": 0.0, "grad_norm": 0.008952063508331776, "learning_rate": 2e-05, "loss": -0.004902808461338282, "num_tokens": 5862133.0, "reward": 0.38715821504592896, "reward_std": 0.5228502154350281, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.2461751252412796, "rewards/length_penalty/std": 0.08678169548511505, "sampling/importance_sampling_ratio/max": 1.7018791437149048, "sampling/importance_sampling_ratio/mean": 0.9946082830429077, "sampling/importance_sampling_ratio/min": 0.6620524525642395, "sampling/sampling_logp_difference/max": 0.5317330360412598, "sampling/sampling_logp_difference/mean": 0.011036716401576996, "step": 107, "step_time": 11.78904084279202 }, { "clip_ratio/high_max": 0.0008622771662582333, "clip_ratio/high_mean": 0.0008622771662582333, "clip_ratio/low_mean": 0.00011014389747288078, "clip_ratio/low_min": 0.00011014389747288078, "clip_ratio/region_mean": 0.0009724210685817525, "completions/clipped_ratio": 0.0, "completions/max_length": 828.0, "completions/max_terminated_length": 828.0, "completions/mean_length": 476.13336181640625, "completions/mean_terminated_length": 476.13336181640625, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 0.24200636645158133, "epoch": 0.15743440233236153, "frac_reward_zero_std": 0.0, "grad_norm": 0.009640194475650787, "learning_rate": 2e-05, "loss": -0.00044576090294867754, "num_tokens": 5895831.0, "reward": 0.10084635764360428, "reward_std": 0.4789772629737854, "rewards/correctness/mean": 0.3333333432674408, "rewards/correctness/std": 0.4753826856613159, "rewards/length_penalty/mean": -0.23248697817325592, "rewards/length_penalty/std": 0.07821419835090637, "sampling/importance_sampling_ratio/max": 1.4278913736343384, "sampling/importance_sampling_ratio/mean": 0.9917312264442444, "sampling/importance_sampling_ratio/min": 0.6590301990509033, "sampling/sampling_logp_difference/max": 0.41698598861694336, "sampling/sampling_logp_difference/mean": 0.015516233630478382, "step": 108, "step_time": 11.562631891807541 }, { "clip_ratio/high_max": 0.0007829671449144371, "clip_ratio/high_mean": 0.0007829671449144371, "clip_ratio/low_mean": 0.0001908329916962733, "clip_ratio/low_min": 0.0001908329916962733, "clip_ratio/region_mean": 0.0009738001293347528, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1752.0, "completions/mean_length": 852.36669921875, "completions/mean_terminated_length": 789.4385986328125, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.29062982896963757, "epoch": 0.1588921282798834, "frac_reward_zero_std": 0.0, "grad_norm": 0.010587343014776707, "learning_rate": 2e-05, "loss": 0.021413404494524002, "num_tokens": 5954923.0, "reward": 0.3338053524494171, "reward_std": 0.5650517344474792, "rewards/correctness/mean": 0.75, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.4161946475505829, "rewards/length_penalty/std": 0.20770102739334106, "sampling/importance_sampling_ratio/max": 1.5148905515670776, "sampling/importance_sampling_ratio/mean": 0.9898743033409119, "sampling/importance_sampling_ratio/min": 0.4112025797367096, "sampling/sampling_logp_difference/max": 0.8886693716049194, "sampling/sampling_logp_difference/mean": 0.01754608564078808, "step": 109, "step_time": 27.17084173578769 }, { "clip_ratio/high_max": 0.0009141297972140213, "clip_ratio/high_mean": 0.0009141297972140213, "clip_ratio/low_mean": 0.00023761014502573138, "clip_ratio/low_min": 0.00023761014502573138, "clip_ratio/region_mean": 0.0011517399252625182, "completions/clipped_ratio": 0.21666668355464935, "completions/max_length": 2048.0, "completions/max_terminated_length": 2007.0, "completions/mean_length": 984.7500610351562, "completions/mean_terminated_length": 690.6595458984375, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 0.4111553480227788, "epoch": 0.16034985422740525, "frac_reward_zero_std": 0.0, "grad_norm": 0.015779634937644005, "learning_rate": 2e-05, "loss": 0.09138652682304382, "num_tokens": 6018318.0, "reward": 0.08583170920610428, "reward_std": 0.7085700631141663, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.4808349609375, "rewards/length_penalty/std": 0.30886977910995483, "sampling/importance_sampling_ratio/max": 1.432719111442566, "sampling/importance_sampling_ratio/mean": 0.9857117533683777, "sampling/importance_sampling_ratio/min": 0.606770396232605, "sampling/sampling_logp_difference/max": 0.49960482120513916, "sampling/sampling_logp_difference/mean": 0.02312241494655609, "step": 110, "step_time": 26.642415746115148 }, { "clip_ratio/high_max": 0.0006274778036943948, "clip_ratio/high_mean": 0.0006274778036943948, "clip_ratio/low_mean": 6.698924213803063e-05, "clip_ratio/low_min": 6.698924213803063e-05, "clip_ratio/region_mean": 0.0006944670434071062, "completions/clipped_ratio": 0.0, "completions/max_length": 987.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 452.41668701171875, "completions/mean_terminated_length": 452.41668701171875, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.19472946474949518, "epoch": 0.1618075801749271, "frac_reward_zero_std": 0.0, "grad_norm": 0.009772730059921741, "learning_rate": 2e-05, "loss": -0.005842759273946285, "num_tokens": 6050023.0, "reward": 0.6957601308822632, "reward_std": 0.3222009241580963, "rewards/correctness/mean": 0.9166666865348816, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.2209065705537796, "rewards/length_penalty/std": 0.09787384420633316, "sampling/importance_sampling_ratio/max": 1.421118140220642, "sampling/importance_sampling_ratio/mean": 0.9931695461273193, "sampling/importance_sampling_ratio/min": 0.6763921976089478, "sampling/sampling_logp_difference/max": 0.39098215103149414, "sampling/sampling_logp_difference/mean": 0.012520139105618, "step": 111, "step_time": 13.254243111237884 }, { "clip_ratio/high_max": 0.0005047594289256571, "clip_ratio/high_mean": 0.0005047594289256571, "clip_ratio/low_mean": 8.132254515658133e-05, "clip_ratio/low_min": 8.132254515658133e-05, "clip_ratio/region_mean": 0.0005860819825708555, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1977.0, "completions/mean_length": 1124.2667236328125, "completions/mean_terminated_length": 1040.2908935546875, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.23988608767588934, "epoch": 0.16326530612244897, "frac_reward_zero_std": 0.0, "grad_norm": 0.014338881708681583, "learning_rate": 2e-05, "loss": 0.05090166628360748, "num_tokens": 6121959.0, "reward": -0.0989583358168602, "reward_std": 0.6507073640823364, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.5489583611488342, "rewards/length_penalty/std": 0.23871323466300964, "sampling/importance_sampling_ratio/max": 1.4403581619262695, "sampling/importance_sampling_ratio/mean": 0.991570770740509, "sampling/importance_sampling_ratio/min": 0.48501986265182495, "sampling/sampling_logp_difference/max": 0.7235654592514038, "sampling/sampling_logp_difference/mean": 0.015152319334447384, "step": 112, "step_time": 27.166342918062583 }, { "clip_ratio/high_max": 0.0011074609986584012, "clip_ratio/high_mean": 0.0011074609986584012, "clip_ratio/low_mean": 4.443851988374566e-05, "clip_ratio/low_min": 4.443851988374566e-05, "clip_ratio/region_mean": 0.001151899530668743, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 757.2667236328125, "completions/mean_terminated_length": 712.7586059570312, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.2371274953087171, "epoch": 0.16472303206997085, "frac_reward_zero_std": 0.0, "grad_norm": 0.012095651589334011, "learning_rate": 2e-05, "loss": 0.046309418976306915, "num_tokens": 6171975.0, "reward": 0.3469075560569763, "reward_std": 0.4822749197483063, "rewards/correctness/mean": 0.7166666388511658, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.36975911259651184, "rewards/length_penalty/std": 0.18612240254878998, "sampling/importance_sampling_ratio/max": 1.420120120048523, "sampling/importance_sampling_ratio/mean": 0.9917676448822021, "sampling/importance_sampling_ratio/min": 0.5245876908302307, "sampling/sampling_logp_difference/max": 0.645142674446106, "sampling/sampling_logp_difference/mean": 0.014648997224867344, "step": 113, "step_time": 26.096502298722044 }, { "clip_ratio/high_max": 0.0010038233691981684, "clip_ratio/high_mean": 0.0010038233691981684, "clip_ratio/low_mean": 9.734569539432414e-05, "clip_ratio/low_min": 9.734569539432414e-05, "clip_ratio/region_mean": 0.0011011690367013216, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1854.0, "completions/mean_length": 923.4833984375, "completions/mean_terminated_length": 750.4807739257812, "completions/min_length": 462.0, "completions/min_terminated_length": 462.0, "entropy": 0.30406248321135837, "epoch": 0.1661807580174927, "frac_reward_zero_std": 0.0, "grad_norm": 0.010896090418100357, "learning_rate": 2e-05, "loss": -0.005914328619837761, "num_tokens": 6231604.0, "reward": 0.3490804135799408, "reward_std": 0.6274470090866089, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.45091959834098816, "rewards/length_penalty/std": 0.249985471367836, "sampling/importance_sampling_ratio/max": 1.4471254348754883, "sampling/importance_sampling_ratio/mean": 0.9893328547477722, "sampling/importance_sampling_ratio/min": 0.6689406037330627, "sampling/sampling_logp_difference/max": 0.40206003189086914, "sampling/sampling_logp_difference/mean": 0.018006250262260437, "step": 114, "step_time": 26.676251277094707 }, { "clip_ratio/high_max": 0.00043013242005448166, "clip_ratio/high_mean": 0.00043013242005448166, "clip_ratio/low_mean": 0.000206737793632783, "clip_ratio/low_min": 0.000206737793632783, "clip_ratio/region_mean": 0.0006368702112619454, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 1806.0, "completions/mean_length": 1120.283447265625, "completions/mean_terminated_length": 782.9318237304688, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.3369167347749074, "epoch": 0.16763848396501457, "frac_reward_zero_std": 0.0, "grad_norm": 0.01141158677637577, "learning_rate": 2e-05, "loss": 0.044828686863183975, "num_tokens": 6304291.0, "reward": -0.11368001997470856, "reward_std": 0.7972869873046875, "rewards/correctness/mean": 0.4333333373069763, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.5470133423805237, "rewards/length_penalty/std": 0.345553994178772, "sampling/importance_sampling_ratio/max": 1.459780216217041, "sampling/importance_sampling_ratio/mean": 0.9883002638816833, "sampling/importance_sampling_ratio/min": 0.5899421572685242, "sampling/sampling_logp_difference/max": 0.5277307033538818, "sampling/sampling_logp_difference/mean": 0.01965204067528248, "step": 115, "step_time": 27.795288962312043 }, { "clip_ratio/high_max": 0.0004979478107998148, "clip_ratio/high_mean": 0.0004979478107998148, "clip_ratio/low_mean": 0.00015590013936161995, "clip_ratio/low_min": 0.00015590013936161995, "clip_ratio/region_mean": 0.0006538479428854771, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 1970.0, "completions/mean_length": 1126.416748046875, "completions/mean_terminated_length": 791.2954711914062, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "entropy": 0.3682697092493375, "epoch": 0.16909620991253643, "frac_reward_zero_std": 0.0, "grad_norm": 0.016534850001335144, "learning_rate": 2e-05, "loss": 0.0898706391453743, "num_tokens": 6376686.0, "reward": 0.01665852963924408, "reward_std": 0.7991631627082825, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.5500081181526184, "rewards/length_penalty/std": 0.3267991542816162, "sampling/importance_sampling_ratio/max": 1.4670318365097046, "sampling/importance_sampling_ratio/mean": 0.9871171712875366, "sampling/importance_sampling_ratio/min": 0.6514776945114136, "sampling/sampling_logp_difference/max": 0.4285120964050293, "sampling/sampling_logp_difference/mean": 0.02166234888136387, "step": 116, "step_time": 27.74404573487118 }, { "clip_ratio/high_max": 0.0008037452256151786, "clip_ratio/high_mean": 0.0008037452256151786, "clip_ratio/low_mean": 2.661556451736639e-05, "clip_ratio/low_min": 2.661556451736639e-05, "clip_ratio/region_mean": 0.0008303607901325449, "completions/clipped_ratio": 0.0, "completions/max_length": 1107.0, "completions/max_terminated_length": 1107.0, "completions/mean_length": 606.6500244140625, "completions/mean_terminated_length": 606.6500244140625, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.2057554523150126, "epoch": 0.17055393586005832, "frac_reward_zero_std": 0.0, "grad_norm": 0.010343342088162899, "learning_rate": 2e-05, "loss": 0.009249605238437653, "num_tokens": 6418125.0, "reward": 0.5037841796875, "reward_std": 0.39930668473243713, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.29621583223342896, "rewards/length_penalty/std": 0.09691675752401352, "sampling/importance_sampling_ratio/max": 1.4326097965240479, "sampling/importance_sampling_ratio/mean": 0.9929855465888977, "sampling/importance_sampling_ratio/min": 0.657730758190155, "sampling/sampling_logp_difference/max": 0.4189596176147461, "sampling/sampling_logp_difference/mean": 0.013125832192599773, "step": 117, "step_time": 14.536963875871152 }, { "clip_ratio/high_max": 0.0009456396231447192, "clip_ratio/high_mean": 0.0009456396231447192, "clip_ratio/low_mean": 6.059137134191891e-05, "clip_ratio/low_min": 6.059137134191891e-05, "clip_ratio/region_mean": 0.0010062309932739784, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1401.0, "completions/mean_length": 577.9500122070312, "completions/mean_terminated_length": 500.5789489746094, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.25233660886685055, "epoch": 0.17201166180758018, "frac_reward_zero_std": 0.0, "grad_norm": 0.0110774589702487, "learning_rate": 2e-05, "loss": 0.04032396152615547, "num_tokens": 6457122.0, "reward": 0.41779786348342896, "reward_std": 0.6189420819282532, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.2822021543979645, "rewards/length_penalty/std": 0.2023358941078186, "sampling/importance_sampling_ratio/max": 1.4071640968322754, "sampling/importance_sampling_ratio/mean": 0.991232693195343, "sampling/importance_sampling_ratio/min": 0.5602762699127197, "sampling/sampling_logp_difference/max": 0.5793251991271973, "sampling/sampling_logp_difference/mean": 0.016044238582253456, "step": 118, "step_time": 24.539006631355733 }, { "clip_ratio/high_max": 0.0005012461285029227, "clip_ratio/high_mean": 0.0005012461285029227, "clip_ratio/low_mean": 0.00011166577314725146, "clip_ratio/low_min": 0.00011166577314725146, "clip_ratio/region_mean": 0.0006129119089261318, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1698.0, "completions/mean_length": 696.4000244140625, "completions/mean_terminated_length": 488.4615478515625, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 0.22835732996463776, "epoch": 0.17346938775510204, "frac_reward_zero_std": 0.0, "grad_norm": 0.01016679685562849, "learning_rate": 2e-05, "loss": 0.010078591294586658, "num_tokens": 6504116.0, "reward": 0.3099609613418579, "reward_std": 0.6975182890892029, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.34003907442092896, "rewards/length_penalty/std": 0.2913699746131897, "sampling/importance_sampling_ratio/max": 1.4403685331344604, "sampling/importance_sampling_ratio/mean": 0.991732120513916, "sampling/importance_sampling_ratio/min": 0.5591676831245422, "sampling/sampling_logp_difference/max": 0.5813058614730835, "sampling/sampling_logp_difference/mean": 0.01446867547929287, "step": 119, "step_time": 26.685395759763196 }, { "clip_ratio/high_max": 0.0009717244538478553, "clip_ratio/high_mean": 0.0009717244538478553, "clip_ratio/low_mean": 0.00020052308173035271, "clip_ratio/low_min": 0.00020052308173035271, "clip_ratio/region_mean": 0.0011722475367908676, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 2031.0, "completions/mean_length": 923.050048828125, "completions/mean_terminated_length": 798.0555419921875, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 0.23362419505914053, "epoch": 0.1749271137026239, "frac_reward_zero_std": 0.0, "grad_norm": 0.015248598530888557, "learning_rate": 2e-05, "loss": 0.05209886655211449, "num_tokens": 6563929.0, "reward": 0.03262532874941826, "reward_std": 0.6359132528305054, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.4507080018520355, "rewards/length_penalty/std": 0.2893427610397339, "sampling/importance_sampling_ratio/max": 2.494029998779297, "sampling/importance_sampling_ratio/mean": 0.9919755458831787, "sampling/importance_sampling_ratio/min": 0.6522249579429626, "sampling/sampling_logp_difference/max": 0.9138998985290527, "sampling/sampling_logp_difference/mean": 0.014302060939371586, "step": 120, "step_time": 26.535727652022615 }, { "clip_ratio/high_max": 0.0004296866354707163, "clip_ratio/high_mean": 0.0004296866354707163, "clip_ratio/low_mean": 0.00015267804944111654, "clip_ratio/low_min": 0.00015267804944111654, "clip_ratio/region_mean": 0.0005823646982510885, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1350.0, "completions/mean_length": 783.7833862304688, "completions/mean_terminated_length": 560.686279296875, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.4176936869819959, "epoch": 0.17638483965014579, "frac_reward_zero_std": 0.0, "grad_norm": 0.008688746020197868, "learning_rate": 2e-05, "loss": 0.001725086010992527, "num_tokens": 6614916.0, "reward": 0.08395996689796448, "reward_std": 0.6422682404518127, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.3827067017555237, "rewards/length_penalty/std": 0.27486416697502136, "sampling/importance_sampling_ratio/max": 1.4164466857910156, "sampling/importance_sampling_ratio/mean": 0.9856937527656555, "sampling/importance_sampling_ratio/min": 0.5565100908279419, "sampling/sampling_logp_difference/max": 0.5860700607299805, "sampling/sampling_logp_difference/mean": 0.023755677044391632, "step": 121, "step_time": 25.770300252130255 }, { "clip_ratio/high_max": 0.0004953260383141848, "clip_ratio/high_mean": 0.0004953260383141848, "clip_ratio/low_mean": 0.00020123962652481472, "clip_ratio/low_min": 0.00020123962652481472, "clip_ratio/region_mean": 0.000696565669689638, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 756.4667358398438, "completions/mean_terminated_length": 639.0545043945312, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.3642074267069499, "epoch": 0.17784256559766765, "frac_reward_zero_std": 0.0, "grad_norm": 0.01703622192144394, "learning_rate": 2e-05, "loss": 0.025002527981996536, "num_tokens": 6665144.0, "reward": 0.21396484971046448, "reward_std": 0.6749217510223389, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.3693684935569763, "rewards/length_penalty/std": 0.24041447043418884, "sampling/importance_sampling_ratio/max": 1.4403629302978516, "sampling/importance_sampling_ratio/mean": 0.9874864816665649, "sampling/importance_sampling_ratio/min": 0.5955517888069153, "sampling/sampling_logp_difference/max": 0.5182669162750244, "sampling/sampling_logp_difference/mean": 0.02108372375369072, "step": 122, "step_time": 26.300538864918053 }, { "clip_ratio/high_max": 0.0011148687820726384, "clip_ratio/high_mean": 0.0011148687820726384, "clip_ratio/low_mean": 9.008175159882133e-05, "clip_ratio/low_min": 9.008175159882133e-05, "clip_ratio/region_mean": 0.0012049505409474175, "completions/clipped_ratio": 0.0, "completions/max_length": 1987.0, "completions/max_terminated_length": 1987.0, "completions/mean_length": 766.4500122070312, "completions/mean_terminated_length": 766.4500122070312, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.198521355787913, "epoch": 0.1793002915451895, "frac_reward_zero_std": 0.0, "grad_norm": 0.011132443323731422, "learning_rate": 2e-05, "loss": 0.0012619979679584503, "num_tokens": 6716791.0, "reward": 0.49242353439331055, "reward_std": 0.38170745968818665, "rewards/correctness/mean": 0.8666666746139526, "rewards/correctness/std": 0.34280332922935486, "rewards/length_penalty/mean": -0.3742431700229645, "rewards/length_penalty/std": 0.16473336517810822, "sampling/importance_sampling_ratio/max": 1.439562439918518, "sampling/importance_sampling_ratio/mean": 0.992904782295227, "sampling/importance_sampling_ratio/min": 0.4723667502403259, "sampling/sampling_logp_difference/max": 0.7499996423721313, "sampling/sampling_logp_difference/mean": 0.012830211780965328, "step": 123, "step_time": 24.510049313073978 }, { "clip_ratio/high_max": 0.0006655274919467047, "clip_ratio/high_mean": 0.0006655274919467047, "clip_ratio/low_mean": 0.0003246031459032868, "clip_ratio/low_min": 0.0003246031459032868, "clip_ratio/region_mean": 0.0009901306378499914, "completions/clipped_ratio": 0.21666668355464935, "completions/max_length": 2048.0, "completions/max_terminated_length": 1711.0, "completions/mean_length": 991.7333984375, "completions/mean_terminated_length": 699.574462890625, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.3197648997108142, "epoch": 0.18075801749271136, "frac_reward_zero_std": 0.0, "grad_norm": 0.018542077392339706, "learning_rate": 2e-05, "loss": 0.07707426697015762, "num_tokens": 6780235.0, "reward": 0.01575521007180214, "reward_std": 0.7430059313774109, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.48424479365348816, "rewards/length_penalty/std": 0.32906633615493774, "sampling/importance_sampling_ratio/max": 1.648996353149414, "sampling/importance_sampling_ratio/mean": 0.9885635375976562, "sampling/importance_sampling_ratio/min": 0.6289244890213013, "sampling/sampling_logp_difference/max": 0.5001667737960815, "sampling/sampling_logp_difference/mean": 0.019025621935725212, "step": 124, "step_time": 26.444706220878288 }, { "clip_ratio/high_max": 0.0005606148139728854, "clip_ratio/high_mean": 0.0005606148139728854, "clip_ratio/low_mean": 0.0001834169537081228, "clip_ratio/low_min": 0.0001834169537081228, "clip_ratio/region_mean": 0.0007440317519164333, "completions/clipped_ratio": 0.23333334922790527, "completions/max_length": 2048.0, "completions/max_terminated_length": 1928.0, "completions/mean_length": 1131.7000732421875, "completions/mean_terminated_length": 852.8261108398438, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.2989233136177063, "epoch": 0.18221574344023322, "frac_reward_zero_std": 0.0, "grad_norm": 0.01351092103868723, "learning_rate": 2e-05, "loss": 0.08028208464384079, "num_tokens": 6852627.0, "reward": 0.0974121168255806, "reward_std": 0.7621957063674927, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.5525878667831421, "rewards/length_penalty/std": 0.3241298198699951, "sampling/importance_sampling_ratio/max": 1.4517446756362915, "sampling/importance_sampling_ratio/mean": 0.9894800186157227, "sampling/importance_sampling_ratio/min": 0.6503453254699707, "sampling/sampling_logp_difference/max": 0.4302518367767334, "sampling/sampling_logp_difference/mean": 0.017761768773198128, "step": 125, "step_time": 27.731713354354724 }, { "clip_ratio/high_max": 0.0004232441254619819, "clip_ratio/high_mean": 0.0004232441254619819, "clip_ratio/low_mean": 0.00024141892936313525, "clip_ratio/low_min": 0.00024141892936313525, "clip_ratio/region_mean": 0.0006646630693770325, "completions/clipped_ratio": 0.18333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 1277.916748046875, "completions/mean_terminated_length": 1105.040771484375, "completions/min_length": 419.0, "completions/min_terminated_length": 419.0, "entropy": 0.2783774783213933, "epoch": 0.1836734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.012387122958898544, "learning_rate": 2e-05, "loss": 0.035095829516649246, "num_tokens": 6936512.0, "reward": -0.24064943194389343, "reward_std": 0.7253720164299011, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014302253723, "rewards/length_penalty/mean": -0.6239827275276184, "rewards/length_penalty/std": 0.27273300290107727, "sampling/importance_sampling_ratio/max": 1.6612882614135742, "sampling/importance_sampling_ratio/mean": 0.990386426448822, "sampling/importance_sampling_ratio/min": 0.5785393118858337, "sampling/sampling_logp_difference/max": 0.5472488403320312, "sampling/sampling_logp_difference/mean": 0.016780639067292213, "step": 126, "step_time": 28.712742294883355 }, { "clip_ratio/high_max": 0.0009337971811570848, "clip_ratio/high_mean": 0.0009337971811570848, "clip_ratio/low_mean": 0.0002482736502618839, "clip_ratio/low_min": 0.0002482736502618839, "clip_ratio/region_mean": 0.0011820707974645, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1819.0, "completions/mean_length": 949.2000732421875, "completions/mean_terminated_length": 780.1538696289062, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.4010750949382782, "epoch": 0.18513119533527697, "frac_reward_zero_std": 0.0, "grad_norm": 0.013337875716388226, "learning_rate": 2e-05, "loss": 0.016871880739927292, "num_tokens": 6998284.0, "reward": 0.08652344346046448, "reward_std": 0.6886231303215027, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.4634765684604645, "rewards/length_penalty/std": 0.2748017907142639, "sampling/importance_sampling_ratio/max": 1.4107288122177124, "sampling/importance_sampling_ratio/mean": 0.9860550165176392, "sampling/importance_sampling_ratio/min": 0.6588606238365173, "sampling/sampling_logp_difference/max": 0.41724324226379395, "sampling/sampling_logp_difference/mean": 0.023269442841410637, "step": 127, "step_time": 27.239234886830673 }, { "clip_ratio/high_max": 0.0004995747658540495, "clip_ratio/high_mean": 0.0004995747658540495, "clip_ratio/low_mean": 0.00010617072014914204, "clip_ratio/low_min": 0.00010617072014914204, "clip_ratio/region_mean": 0.0006057454884285107, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1957.0, "completions/mean_length": 951.4000244140625, "completions/mean_terminated_length": 829.5555419921875, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.21109230816364288, "epoch": 0.18658892128279883, "frac_reward_zero_std": 0.0, "grad_norm": 0.012693856842815876, "learning_rate": 2e-05, "loss": -0.014684738591313362, "num_tokens": 7061468.0, "reward": -0.28121745586395264, "reward_std": 0.5295966267585754, "rewards/correctness/mean": 0.18333333730697632, "rewards/correctness/std": 0.39020493626594543, "rewards/length_penalty/mean": -0.46455079317092896, "rewards/length_penalty/std": 0.2660280466079712, "sampling/importance_sampling_ratio/max": 1.4671049118041992, "sampling/importance_sampling_ratio/mean": 0.9926087260246277, "sampling/importance_sampling_ratio/min": 0.5823831558227539, "sampling/sampling_logp_difference/max": 0.5406267642974854, "sampling/sampling_logp_difference/mean": 0.013477679342031479, "step": 128, "step_time": 28.212278296938166 }, { "clip_ratio/high_max": 0.0009754830462043174, "clip_ratio/high_mean": 0.0009754830462043174, "clip_ratio/low_mean": 8.977870796419059e-05, "clip_ratio/low_min": 8.977870796419059e-05, "clip_ratio/region_mean": 0.0010652617468925503, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1830.0, "completions/mean_length": 774.7500610351562, "completions/mean_terminated_length": 659.0, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.34729481240113574, "epoch": 0.1880466472303207, "frac_reward_zero_std": 0.0, "grad_norm": 0.009247354231774807, "learning_rate": 2e-05, "loss": -0.005573976784944534, "num_tokens": 7112833.0, "reward": -0.17829591035842896, "reward_std": 0.524168074131012, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.3782958984375, "rewards/length_penalty/std": 0.24723808467388153, "sampling/importance_sampling_ratio/max": 1.4727504253387451, "sampling/importance_sampling_ratio/mean": 0.9879551529884338, "sampling/importance_sampling_ratio/min": 0.6780223250389099, "sampling/sampling_logp_difference/max": 0.38857507705688477, "sampling/sampling_logp_difference/mean": 0.0211569145321846, "step": 129, "step_time": 25.679753988282755 }, { "clip_ratio/high_max": 0.0009580096278417235, "clip_ratio/high_mean": 0.0009580096278417235, "clip_ratio/low_mean": 6.613554554254127e-05, "clip_ratio/low_min": 6.613554554254127e-05, "clip_ratio/region_mean": 0.0010241451770222436, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 2010.0, "completions/mean_length": 875.36669921875, "completions/mean_terminated_length": 791.607177734375, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.29244259744882584, "epoch": 0.18950437317784258, "frac_reward_zero_std": 0.0, "grad_norm": 0.015463539399206638, "learning_rate": 2e-05, "loss": 0.045666784048080444, "num_tokens": 7171605.0, "reward": 0.2392415553331375, "reward_std": 0.6368379592895508, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.4274251163005829, "rewards/length_penalty/std": 0.26480501890182495, "sampling/importance_sampling_ratio/max": 1.4454598426818848, "sampling/importance_sampling_ratio/mean": 0.9897004961967468, "sampling/importance_sampling_ratio/min": 0.6564763188362122, "sampling/sampling_logp_difference/max": 0.4208686351776123, "sampling/sampling_logp_difference/mean": 0.01750544086098671, "step": 130, "step_time": 27.429522439837456 }, { "clip_ratio/high_max": 0.0009249262996794035, "clip_ratio/high_mean": 0.0009249262996794035, "clip_ratio/low_mean": 0.000166456480656052, "clip_ratio/low_min": 0.000166456480656052, "clip_ratio/region_mean": 0.0010913827815481152, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1916.0, "completions/mean_length": 852.7333984375, "completions/mean_terminated_length": 694.867919921875, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.3944722463687261, "epoch": 0.19096209912536444, "frac_reward_zero_std": 0.0, "grad_norm": 0.0137899499386549, "learning_rate": 2e-05, "loss": 0.021337302401661873, "num_tokens": 7227849.0, "reward": 0.18362630903720856, "reward_std": 0.6749227046966553, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.41637369990348816, "rewards/length_penalty/std": 0.27475234866142273, "sampling/importance_sampling_ratio/max": 1.431358814239502, "sampling/importance_sampling_ratio/mean": 0.9860085248947144, "sampling/importance_sampling_ratio/min": 0.6481273174285889, "sampling/sampling_logp_difference/max": 0.4336681365966797, "sampling/sampling_logp_difference/mean": 0.023906156420707703, "step": 131, "step_time": 26.295353600988165 }, { "clip_ratio/high_max": 0.0013289499426415812, "clip_ratio/high_mean": 0.0013289499426415812, "clip_ratio/low_mean": 3.034164789520825e-05, "clip_ratio/low_min": 3.034164789520825e-05, "clip_ratio/region_mean": 0.0013592915784101933, "completions/clipped_ratio": 0.0, "completions/max_length": 1136.0, "completions/max_terminated_length": 1136.0, "completions/mean_length": 498.45001220703125, "completions/mean_terminated_length": 498.45001220703125, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.20351594934860864, "epoch": 0.1924198250728863, "frac_reward_zero_std": 0.0, "grad_norm": 0.009152635000646114, "learning_rate": 2e-05, "loss": -0.0025878362357616425, "num_tokens": 7264976.0, "reward": 0.6899495720863342, "reward_std": 0.289070725440979, "rewards/correctness/mean": 0.9333333373069763, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.24338379502296448, "rewards/length_penalty/std": 0.12275136262178421, "sampling/importance_sampling_ratio/max": 1.4464099407196045, "sampling/importance_sampling_ratio/mean": 0.9926159381866455, "sampling/importance_sampling_ratio/min": 0.6483365893363953, "sampling/sampling_logp_difference/max": 0.43334531784057617, "sampling/sampling_logp_difference/mean": 0.013126187026500702, "step": 132, "step_time": 15.227134235436097 }, { "clip_ratio/high_max": 0.0009851352369878441, "clip_ratio/high_mean": 0.0009851352369878441, "clip_ratio/low_mean": 6.253908698757489e-05, "clip_ratio/low_min": 6.253908698757489e-05, "clip_ratio/region_mean": 0.001047674323975419, "completions/clipped_ratio": 0.0, "completions/max_length": 1509.0, "completions/max_terminated_length": 1509.0, "completions/mean_length": 743.1000366210938, "completions/mean_terminated_length": 743.1000366210938, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.14064423739910126, "epoch": 0.19387755102040816, "frac_reward_zero_std": 0.0, "grad_norm": 0.011004043743014336, "learning_rate": 2e-05, "loss": -0.014399453997612, "num_tokens": 7314702.0, "reward": 0.40382489562034607, "reward_std": 0.4183441996574402, "rewards/correctness/mean": 0.7666666507720947, "rewards/correctness/std": 0.42652183771133423, "rewards/length_penalty/mean": -0.36284178495407104, "rewards/length_penalty/std": 0.18375426530838013, "sampling/importance_sampling_ratio/max": 1.448881983757019, "sampling/importance_sampling_ratio/mean": 0.9950322508811951, "sampling/importance_sampling_ratio/min": 0.6690908670425415, "sampling/sampling_logp_difference/max": 0.40183544158935547, "sampling/sampling_logp_difference/mean": 0.009434840641915798, "step": 133, "step_time": 19.896736015100032 }, { "clip_ratio/high_max": 0.0009553819812329797, "clip_ratio/high_mean": 0.0009553819812329797, "clip_ratio/low_mean": 0.00016222599030394727, "clip_ratio/low_min": 0.00016222599030394727, "clip_ratio/region_mean": 0.0011176079569850117, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1968.0, "completions/mean_length": 1053.283447265625, "completions/mean_terminated_length": 877.7451171875, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.3020109136899312, "epoch": 0.19533527696793002, "frac_reward_zero_std": 0.0, "grad_norm": 0.01610954850912094, "learning_rate": 2e-05, "loss": 0.03780611976981163, "num_tokens": 7385539.0, "reward": 0.0523681677877903, "reward_std": 0.7675083875656128, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.5142984986305237, "rewards/length_penalty/std": 0.32769298553466797, "sampling/importance_sampling_ratio/max": 1.450823187828064, "sampling/importance_sampling_ratio/mean": 0.9897510409355164, "sampling/importance_sampling_ratio/min": 0.3544185757637024, "sampling/sampling_logp_difference/max": 1.0372766256332397, "sampling/sampling_logp_difference/mean": 0.017968930304050446, "step": 134, "step_time": 28.227011064067483 }, { "clip_ratio/high_max": 0.000593315263055653, "clip_ratio/high_mean": 0.000593315263055653, "clip_ratio/low_mean": 0.0001034647757478524, "clip_ratio/low_min": 0.0001034647757478524, "clip_ratio/region_mean": 0.0006967800363781862, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 796.8833618164062, "completions/mean_terminated_length": 631.6415405273438, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.3218641405304273, "epoch": 0.1967930029154519, "frac_reward_zero_std": 0.0, "grad_norm": 0.011188547126948833, "learning_rate": 2e-05, "loss": 0.026850301772356033, "num_tokens": 7436682.0, "reward": 0.2108968198299408, "reward_std": 0.7179157137870789, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.3891032040119171, "rewards/length_penalty/std": 0.33137357234954834, "sampling/importance_sampling_ratio/max": 1.3966896533966064, "sampling/importance_sampling_ratio/mean": 0.989035427570343, "sampling/importance_sampling_ratio/min": 0.6542773842811584, "sampling/sampling_logp_difference/max": 0.4242238998413086, "sampling/sampling_logp_difference/mean": 0.019122382625937462, "step": 135, "step_time": 25.719649199862033 }, { "clip_ratio/high_max": 0.0011213517136638984, "clip_ratio/high_mean": 0.0011213517136638984, "clip_ratio/low_mean": 8.593292295699939e-05, "clip_ratio/low_min": 8.593292295699939e-05, "clip_ratio/region_mean": 0.0012072846293449402, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1966.0, "completions/mean_length": 651.3333740234375, "completions/mean_terminated_length": 603.1724243164062, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.42364654938379925, "epoch": 0.19825072886297376, "frac_reward_zero_std": 0.0, "grad_norm": 0.014928486198186874, "learning_rate": 2e-05, "loss": -0.004173014312982559, "num_tokens": 7481172.0, "reward": 0.4319661557674408, "reward_std": 0.5589545369148254, "rewards/correctness/mean": 0.75, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.3180338442325592, "rewards/length_penalty/std": 0.219534769654274, "sampling/importance_sampling_ratio/max": 1.714572787284851, "sampling/importance_sampling_ratio/mean": 0.985823392868042, "sampling/importance_sampling_ratio/min": 0.6611449718475342, "sampling/sampling_logp_difference/max": 0.5391639471054077, "sampling/sampling_logp_difference/mean": 0.024284308776259422, "step": 136, "step_time": 26.223453251179308 }, { "clip_ratio/high_max": 0.001284141922951676, "clip_ratio/high_mean": 0.001284141922951676, "clip_ratio/low_mean": 2.4341560977821548e-05, "clip_ratio/low_min": 2.4341560977821548e-05, "clip_ratio/region_mean": 0.0013084834742282208, "completions/clipped_ratio": 0.0, "completions/max_length": 1802.0, "completions/max_terminated_length": 1802.0, "completions/mean_length": 617.5333862304688, "completions/mean_terminated_length": 617.5333862304688, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.23057991514603296, "epoch": 0.19970845481049562, "frac_reward_zero_std": 0.0, "grad_norm": 0.011205561459064484, "learning_rate": 2e-05, "loss": -0.0336463563144207, "num_tokens": 7522004.0, "reward": 0.48180341720581055, "reward_std": 0.49382373690605164, "rewards/correctness/mean": 0.7833333611488342, "rewards/correctness/std": 0.41545021533966064, "rewards/length_penalty/mean": -0.3015299439430237, "rewards/length_penalty/std": 0.15826867520809174, "sampling/importance_sampling_ratio/max": 1.5545979738235474, "sampling/importance_sampling_ratio/mean": 0.9920415282249451, "sampling/importance_sampling_ratio/min": 0.6387434601783752, "sampling/sampling_logp_difference/max": 0.44825243949890137, "sampling/sampling_logp_difference/mean": 0.014765698462724686, "step": 137, "step_time": 21.748775377636775 }, { "clip_ratio/high_max": 0.0005354360716106991, "clip_ratio/high_mean": 0.0005354360716106991, "clip_ratio/low_mean": 2.5948414986487478e-05, "clip_ratio/low_min": 2.5948414986487478e-05, "clip_ratio/region_mean": 0.0005613844841718674, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 647.5000610351562, "completions/mean_terminated_length": 623.7626953125, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.21775240947802862, "epoch": 0.20116618075801748, "frac_reward_zero_std": 0.0, "grad_norm": 0.010806845501065254, "learning_rate": 2e-05, "loss": -0.007559105753898621, "num_tokens": 7564524.0, "reward": 0.30050456523895264, "reward_std": 0.5465244650840759, "rewards/correctness/mean": 0.6166666746139526, "rewards/correctness/std": 0.4903014302253723, "rewards/length_penalty/mean": -0.316162109375, "rewards/length_penalty/std": 0.1294468343257904, "sampling/importance_sampling_ratio/max": 1.4604040384292603, "sampling/importance_sampling_ratio/mean": 0.9926302433013916, "sampling/importance_sampling_ratio/min": 0.5560197234153748, "sampling/sampling_logp_difference/max": 0.586951494216919, "sampling/sampling_logp_difference/mean": 0.014007749035954475, "step": 138, "step_time": 24.29082035413012 }, { "clip_ratio/high_max": 0.0009670745542583367, "clip_ratio/high_mean": 0.0009670745542583367, "clip_ratio/low_mean": 0.000139080656178218, "clip_ratio/low_min": 0.000139080656178218, "clip_ratio/region_mean": 0.001106155231051768, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1682.0, "completions/mean_length": 747.7833862304688, "completions/mean_terminated_length": 679.3508911132812, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.23754004389047623, "epoch": 0.20262390670553937, "frac_reward_zero_std": 0.0, "grad_norm": 0.01702095754444599, "learning_rate": 2e-05, "loss": 0.018035396933555603, "num_tokens": 7613141.0, "reward": 0.33487144112586975, "reward_std": 0.5780368447303772, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.3651285767555237, "rewards/length_penalty/std": 0.2624134123325348, "sampling/importance_sampling_ratio/max": 1.4660247564315796, "sampling/importance_sampling_ratio/mean": 0.991218090057373, "sampling/importance_sampling_ratio/min": 0.6518484354019165, "sampling/sampling_logp_difference/max": 0.42794322967529297, "sampling/sampling_logp_difference/mean": 0.015252550132572651, "step": 139, "step_time": 25.713777780067176 }, { "clip_ratio/high_max": 0.001041515274361397, "clip_ratio/high_mean": 0.001041515274361397, "clip_ratio/low_mean": 8.830907124017055e-05, "clip_ratio/low_min": 8.830907124017055e-05, "clip_ratio/region_mean": 0.0011298243383256097, "completions/clipped_ratio": 0.0, "completions/max_length": 1405.0, "completions/max_terminated_length": 1405.0, "completions/mean_length": 550.9666748046875, "completions/mean_terminated_length": 550.9666748046875, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.20439065992832184, "epoch": 0.20408163265306123, "frac_reward_zero_std": 0.0, "grad_norm": 0.010910374112427235, "learning_rate": 2e-05, "loss": -0.002087140455842018, "num_tokens": 7650519.0, "reward": 0.6976400017738342, "reward_std": 0.2075270414352417, "rewards/correctness/mean": 0.9666666388511658, "rewards/correctness/std": 0.1810203343629837, "rewards/length_penalty/mean": -0.2690266966819763, "rewards/length_penalty/std": 0.11686760932207108, "sampling/importance_sampling_ratio/max": 1.4039456844329834, "sampling/importance_sampling_ratio/mean": 0.9930278062820435, "sampling/importance_sampling_ratio/min": 0.6248584389686584, "sampling/sampling_logp_difference/max": 0.4702301025390625, "sampling/sampling_logp_difference/mean": 0.013711715117096901, "step": 140, "step_time": 17.20439298520796 }, { "clip_ratio/high_max": 0.00038998989960722, "clip_ratio/high_mean": 0.00038998989960722, "clip_ratio/low_mean": 0.000349673655970643, "clip_ratio/low_min": 0.000349673655970643, "clip_ratio/region_mean": 0.0007396635483019054, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1954.0, "completions/mean_length": 926.4667358398438, "completions/mean_terminated_length": 728.549072265625, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.36274297535419464, "epoch": 0.2055393586005831, "frac_reward_zero_std": 0.0, "grad_norm": 0.01296077761799097, "learning_rate": 2e-05, "loss": 0.06322832405567169, "num_tokens": 7710807.0, "reward": 0.014290365390479565, "reward_std": 0.7337704300880432, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.4523763060569763, "rewards/length_penalty/std": 0.3023598790168762, "sampling/importance_sampling_ratio/max": 1.8982439041137695, "sampling/importance_sampling_ratio/mean": 0.9872742891311646, "sampling/importance_sampling_ratio/min": 0.5476210713386536, "sampling/sampling_logp_difference/max": 0.6409292221069336, "sampling/sampling_logp_difference/mean": 0.02172762341797352, "step": 141, "step_time": 26.343239391921088 }, { "clip_ratio/high_max": 0.0005487072364000293, "clip_ratio/high_mean": 0.0005487072364000293, "clip_ratio/low_mean": 0.0002699390655228247, "clip_ratio/low_min": 0.0002699390655228247, "clip_ratio/region_mean": 0.0008186462994975349, "completions/clipped_ratio": 0.3500000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 1864.0, "completions/mean_length": 1286.5001220703125, "completions/mean_terminated_length": 876.4615478515625, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.3190828462441762, "epoch": 0.20699708454810495, "frac_reward_zero_std": 0.0, "grad_norm": 0.019596340134739876, "learning_rate": 2e-05, "loss": 0.07146541029214859, "num_tokens": 7794597.0, "reward": -0.34484052658081055, "reward_std": 0.5870059728622437, "rewards/correctness/mean": 0.28333333134651184, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.628173828125, "rewards/length_penalty/std": 0.32059621810913086, "sampling/importance_sampling_ratio/max": 2.1116273403167725, "sampling/importance_sampling_ratio/mean": 0.9887951016426086, "sampling/importance_sampling_ratio/min": 0.30148351192474365, "sampling/sampling_logp_difference/max": 1.1990399360656738, "sampling/sampling_logp_difference/mean": 0.019383693113923073, "step": 142, "step_time": 28.993721147766337 }, { "clip_ratio/high_max": 0.0006226475985992389, "clip_ratio/high_mean": 0.0006226475985992389, "clip_ratio/low_mean": 9.212004079017788e-05, "clip_ratio/low_min": 9.212004079017788e-05, "clip_ratio/region_mean": 0.0007147676418147361, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 2007.0, "completions/mean_length": 1030.4833984375, "completions/mean_terminated_length": 937.9818115234375, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "entropy": 0.32883496085802716, "epoch": 0.20845481049562684, "frac_reward_zero_std": 0.0, "grad_norm": 0.01710645481944084, "learning_rate": 2e-05, "loss": 0.01035105250775814, "num_tokens": 7861916.0, "reward": -0.3031657040119171, "reward_std": 0.4475523829460144, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.5031656622886658, "rewards/length_penalty/std": 0.25336435437202454, "sampling/importance_sampling_ratio/max": 1.4255783557891846, "sampling/importance_sampling_ratio/mean": 0.9885072112083435, "sampling/importance_sampling_ratio/min": 0.5910841822624207, "sampling/sampling_logp_difference/max": 0.5257967710494995, "sampling/sampling_logp_difference/mean": 0.01998257078230381, "step": 143, "step_time": 26.875178738264367 }, { "clip_ratio/high_max": 0.0009803994495693284, "clip_ratio/high_mean": 0.0009803994495693284, "clip_ratio/low_mean": 0.00010792129614856094, "clip_ratio/low_min": 0.00010792129614856094, "clip_ratio/region_mean": 0.001088320719039378, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1951.0, "completions/mean_length": 718.800048828125, "completions/mean_terminated_length": 571.1111450195312, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.3294060428937276, "epoch": 0.2099125364431487, "frac_reward_zero_std": 0.0, "grad_norm": 0.008123371750116348, "learning_rate": 2e-05, "loss": 0.004980289377272129, "num_tokens": 7909194.0, "reward": 0.44902345538139343, "reward_std": 0.6170951724052429, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.3509765565395355, "rewards/length_penalty/std": 0.25817549228668213, "sampling/importance_sampling_ratio/max": 1.4503473043441772, "sampling/importance_sampling_ratio/mean": 0.9886658191680908, "sampling/importance_sampling_ratio/min": 0.6502314805984497, "sampling/sampling_logp_difference/max": 0.43042683601379395, "sampling/sampling_logp_difference/mean": 0.019626792520284653, "step": 144, "step_time": 25.531999272527173 }, { "clip_ratio/high_max": 0.0009709181807314357, "clip_ratio/high_mean": 0.0009709181807314357, "clip_ratio/low_mean": 0.00015339334883416692, "clip_ratio/low_min": 0.00015339334883416692, "clip_ratio/region_mean": 0.0011243115295656025, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1719.0, "completions/mean_length": 876.11669921875, "completions/mean_terminated_length": 792.4107666015625, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.2737928008039792, "epoch": 0.21137026239067055, "frac_reward_zero_std": 0.0, "grad_norm": 0.013971668668091297, "learning_rate": 2e-05, "loss": 0.05236930772662163, "num_tokens": 7966161.0, "reward": 0.2388753443956375, "reward_std": 0.6455968022346497, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.4277913272380829, "rewards/length_penalty/std": 0.2562803328037262, "sampling/importance_sampling_ratio/max": 1.402762532234192, "sampling/importance_sampling_ratio/mean": 0.9905370473861694, "sampling/importance_sampling_ratio/min": 0.4719122052192688, "sampling/sampling_logp_difference/max": 0.7509623765945435, "sampling/sampling_logp_difference/mean": 0.01711660996079445, "step": 145, "step_time": 26.461914269486442 }, { "clip_ratio/high_max": 0.0006462544382278187, "clip_ratio/high_mean": 0.0006462544382278187, "clip_ratio/low_mean": 0.0002863618453072074, "clip_ratio/low_min": 0.0002863618453072074, "clip_ratio/region_mean": 0.0009326162786843876, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1960.0, "completions/mean_length": 709.1666870117188, "completions/mean_terminated_length": 503.19232177734375, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.40346813201904297, "epoch": 0.21282798833819241, "frac_reward_zero_std": 0.0, "grad_norm": 0.009171973913908005, "learning_rate": 2e-05, "loss": 0.0015175435692071915, "num_tokens": 8012971.0, "reward": 0.20372723042964935, "reward_std": 0.679226815700531, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.3462727963924408, "rewards/length_penalty/std": 0.2919903099536896, "sampling/importance_sampling_ratio/max": 1.4315117597579956, "sampling/importance_sampling_ratio/mean": 0.9860925078392029, "sampling/importance_sampling_ratio/min": 0.6461740136146545, "sampling/sampling_logp_difference/max": 0.43668651580810547, "sampling/sampling_logp_difference/mean": 0.02341708354651928, "step": 146, "step_time": 25.89526958204806 }, { "clip_ratio/high_max": 0.000788814082625322, "clip_ratio/high_mean": 0.000788814082625322, "clip_ratio/low_mean": 7.270114535155396e-05, "clip_ratio/low_min": 7.270114535155396e-05, "clip_ratio/region_mean": 0.000861515227976876, "completions/clipped_ratio": 0.0, "completions/max_length": 1075.0, "completions/max_terminated_length": 1075.0, "completions/mean_length": 495.5666809082031, "completions/mean_terminated_length": 495.5666809082031, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.18527658035357794, "epoch": 0.21428571428571427, "frac_reward_zero_std": 0.0, "grad_norm": 0.010867736302316189, "learning_rate": 2e-05, "loss": 0.004002865869551897, "num_tokens": 8046455.0, "reward": 0.42469078302383423, "reward_std": 0.5008008480072021, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.24197591841220856, "rewards/length_penalty/std": 0.13876907527446747, "sampling/importance_sampling_ratio/max": 1.413368821144104, "sampling/importance_sampling_ratio/mean": 0.9933928847312927, "sampling/importance_sampling_ratio/min": 0.6621342301368713, "sampling/sampling_logp_difference/max": 0.41228699684143066, "sampling/sampling_logp_difference/mean": 0.012012646533548832, "step": 147, "step_time": 13.71861265343614 }, { "clip_ratio/high_max": 0.0009729232527509643, "clip_ratio/high_mean": 0.0009729232527509643, "clip_ratio/low_mean": 3.495525743346661e-05, "clip_ratio/low_min": 3.495525743346661e-05, "clip_ratio/region_mean": 0.0010078785150350693, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 769.8167114257812, "completions/mean_terminated_length": 702.5438842773438, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.2404237985610962, "epoch": 0.21574344023323616, "frac_reward_zero_std": 0.0, "grad_norm": 0.028219083324074745, "learning_rate": 2e-05, "loss": 0.11528102308511734, "num_tokens": 8096494.0, "reward": 0.40744632482528687, "reward_std": 0.5218658447265625, "rewards/correctness/mean": 0.7833333611488342, "rewards/correctness/std": 0.41545024514198303, "rewards/length_penalty/mean": -0.37588703632354736, "rewards/length_penalty/std": 0.19455382227897644, "sampling/importance_sampling_ratio/max": 1.4559873342514038, "sampling/importance_sampling_ratio/mean": 0.9913713335990906, "sampling/importance_sampling_ratio/min": 0.6405834555625916, "sampling/sampling_logp_difference/max": 0.445375919342041, "sampling/sampling_logp_difference/mean": 0.014926444739103317, "step": 148, "step_time": 25.856907369336113 }, { "clip_ratio/high_max": 0.0011667126091197133, "clip_ratio/high_mean": 0.0011667126091197133, "clip_ratio/low_mean": 0.00011985406066135813, "clip_ratio/low_min": 0.00011985406066135813, "clip_ratio/region_mean": 0.001286566713436817, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1829.0, "completions/mean_length": 788.2667236328125, "completions/mean_terminated_length": 744.8275756835938, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.28645333151022595, "epoch": 0.21720116618075802, "frac_reward_zero_std": 0.0, "grad_norm": 0.012622080743312836, "learning_rate": 2e-05, "loss": 0.03589104115962982, "num_tokens": 8149690.0, "reward": 0.16510418057441711, "reward_std": 0.5626128911972046, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.38489583134651184, "rewards/length_penalty/std": 0.20437301695346832, "sampling/importance_sampling_ratio/max": 1.4403685331344604, "sampling/importance_sampling_ratio/mean": 0.9900524616241455, "sampling/importance_sampling_ratio/min": 0.5904785990715027, "sampling/sampling_logp_difference/max": 0.5268218517303467, "sampling/sampling_logp_difference/mean": 0.017937393859028816, "step": 149, "step_time": 26.160948349162936 }, { "clip_ratio/high_max": 0.0005951016404045125, "clip_ratio/high_mean": 0.0005951016404045125, "clip_ratio/low_mean": 0.0001770943466302318, "clip_ratio/low_min": 0.0001770943466302318, "clip_ratio/region_mean": 0.0007721959870347442, "completions/clipped_ratio": 0.21666668355464935, "completions/max_length": 2048.0, "completions/max_terminated_length": 1881.0, "completions/mean_length": 930.1000366210938, "completions/mean_terminated_length": 620.8936157226562, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.3673850744962692, "epoch": 0.21865889212827988, "frac_reward_zero_std": 0.0, "grad_norm": 0.015178069472312927, "learning_rate": 2e-05, "loss": -0.0044382475316524506, "num_tokens": 8210576.0, "reward": -0.20415040850639343, "reward_std": 0.6173945665359497, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.45415037870407104, "rewards/length_penalty/std": 0.3532169461250305, "sampling/importance_sampling_ratio/max": 1.442487120628357, "sampling/importance_sampling_ratio/mean": 0.9874463677406311, "sampling/importance_sampling_ratio/min": 0.5390064716339111, "sampling/sampling_logp_difference/max": 0.6180276870727539, "sampling/sampling_logp_difference/mean": 0.02184767834842205, "step": 150, "step_time": 26.75312972185202 }, { "clip_ratio/high_max": 0.0007837310889347767, "clip_ratio/high_mean": 0.0007837310889347767, "clip_ratio/low_mean": 0.00012820728685862073, "clip_ratio/low_min": 0.00012820728685862073, "clip_ratio/region_mean": 0.0009119383757933974, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1938.0, "completions/mean_length": 836.0333862304688, "completions/mean_terminated_length": 749.4642944335938, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.23776968320210776, "epoch": 0.22011661807580174, "frac_reward_zero_std": 0.0, "grad_norm": 0.011509308591485023, "learning_rate": 2e-05, "loss": -0.003738299012184143, "num_tokens": 8264918.0, "reward": 0.12511393427848816, "reward_std": 0.6106396913528442, "rewards/correctness/mean": 0.5333333611488342, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.4082193970680237, "rewards/length_penalty/std": 0.2634795904159546, "sampling/importance_sampling_ratio/max": 1.413131594657898, "sampling/importance_sampling_ratio/mean": 0.9915941953659058, "sampling/importance_sampling_ratio/min": 0.6075857281684875, "sampling/sampling_logp_difference/max": 0.49826204776763916, "sampling/sampling_logp_difference/mean": 0.014894966036081314, "step": 151, "step_time": 26.18424436938949 }, { "clip_ratio/high_max": 0.0011510254771565087, "clip_ratio/high_mean": 0.0011510254771565087, "clip_ratio/low_mean": 0.0002647102919581812, "clip_ratio/low_min": 0.0002647102919581812, "clip_ratio/region_mean": 0.0014157357703273494, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1879.0, "completions/mean_length": 746.4500122070312, "completions/mean_terminated_length": 628.1272583007812, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.4303782532612483, "epoch": 0.22157434402332363, "frac_reward_zero_std": 0.0, "grad_norm": 0.015101080760359764, "learning_rate": 2e-05, "loss": 0.006294237449765205, "num_tokens": 8314415.0, "reward": 0.23552247881889343, "reward_std": 0.6499772667884827, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.3644775450229645, "rewards/length_penalty/std": 0.24947568774223328, "sampling/importance_sampling_ratio/max": 1.4334946870803833, "sampling/importance_sampling_ratio/mean": 0.9842382669448853, "sampling/importance_sampling_ratio/min": 0.6101853251457214, "sampling/sampling_logp_difference/max": 0.49399256706237793, "sampling/sampling_logp_difference/mean": 0.026771733537316322, "step": 152, "step_time": 25.680529301287606 }, { "clip_ratio/high_max": 0.0008009954132527733, "clip_ratio/high_mean": 0.0008009954132527733, "clip_ratio/low_mean": 7.452095572565061e-05, "clip_ratio/low_min": 7.452095572565061e-05, "clip_ratio/region_mean": 0.0008755163774670413, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1615.0, "completions/mean_length": 868.9166870117188, "completions/mean_terminated_length": 806.8596801757812, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.23239794125159582, "epoch": 0.2230320699708455, "frac_reward_zero_std": 0.0, "grad_norm": 0.0174455214291811, "learning_rate": 2e-05, "loss": 0.07516114413738251, "num_tokens": 8372510.0, "reward": 0.1590576320886612, "reward_std": 0.6134790182113647, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.4242757260799408, "rewards/length_penalty/std": 0.1961939036846161, "sampling/importance_sampling_ratio/max": 1.432678461074829, "sampling/importance_sampling_ratio/mean": 0.9919102787971497, "sampling/importance_sampling_ratio/min": 0.6673089265823364, "sampling/sampling_logp_difference/max": 0.40450215339660645, "sampling/sampling_logp_difference/mean": 0.014779479242861271, "step": 153, "step_time": 26.61921369191259 }, { "clip_ratio/high_max": 0.0013186451833462343, "clip_ratio/high_mean": 0.0013186451833462343, "clip_ratio/low_mean": 1.9716865305478375e-05, "clip_ratio/low_min": 1.9716865305478375e-05, "clip_ratio/region_mean": 0.0013383620486517127, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1193.0, "completions/mean_length": 632.5167236328125, "completions/mean_terminated_length": 608.5254516601562, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.19110674659411112, "epoch": 0.22448979591836735, "frac_reward_zero_std": 0.0, "grad_norm": 0.011251011863350868, "learning_rate": 2e-05, "loss": 0.006448462605476379, "num_tokens": 8414591.0, "reward": 0.4911539852619171, "reward_std": 0.46878206729888916, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.30884602665901184, "rewards/length_penalty/std": 0.15812522172927856, "sampling/importance_sampling_ratio/max": 1.4671045541763306, "sampling/importance_sampling_ratio/mean": 0.9933434724807739, "sampling/importance_sampling_ratio/min": 0.6210270524024963, "sampling/sampling_logp_difference/max": 0.4763805866241455, "sampling/sampling_logp_difference/mean": 0.01292010210454464, "step": 154, "step_time": 24.742094105109572 }, { "clip_ratio/high_max": 0.0012121789429026346, "clip_ratio/high_mean": 0.0012121789429026346, "clip_ratio/low_mean": 0.0001811962744492727, "clip_ratio/low_min": 0.0001811962744492727, "clip_ratio/region_mean": 0.0013933752197772264, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1872.0, "completions/mean_length": 729.61669921875, "completions/mean_terminated_length": 707.2711791992188, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.2897387221455574, "epoch": 0.2259475218658892, "frac_reward_zero_std": 0.0, "grad_norm": 0.016924167051911354, "learning_rate": 2e-05, "loss": 0.0368056520819664, "num_tokens": 8463068.0, "reward": 0.32707521319389343, "reward_std": 0.5871769189834595, "rewards/correctness/mean": 0.6833333373069763, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.3562581241130829, "rewards/length_penalty/std": 0.23875026404857635, "sampling/importance_sampling_ratio/max": 1.4597691297531128, "sampling/importance_sampling_ratio/mean": 0.9895320534706116, "sampling/importance_sampling_ratio/min": 0.40709778666496277, "sampling/sampling_logp_difference/max": 0.8987019062042236, "sampling/sampling_logp_difference/mean": 0.019146829843521118, "step": 155, "step_time": 25.909327471861616 }, { "clip_ratio/high_max": 0.0007107193402286308, "clip_ratio/high_mean": 0.0007107193402286308, "clip_ratio/low_mean": 0.00023871100953935334, "clip_ratio/low_min": 0.00023871100953935334, "clip_ratio/region_mean": 0.0009494303424920266, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1929.0, "completions/mean_length": 794.4500122070312, "completions/mean_terminated_length": 628.8867797851562, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 0.3392493550976117, "epoch": 0.22740524781341107, "frac_reward_zero_std": 0.0, "grad_norm": 0.009066332131624222, "learning_rate": 2e-05, "loss": -0.0009449312929064035, "num_tokens": 8514985.0, "reward": 0.2620849609375, "reward_std": 0.7496296763420105, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.3879150450229645, "rewards/length_penalty/std": 0.2956892251968384, "sampling/importance_sampling_ratio/max": 1.6749331951141357, "sampling/importance_sampling_ratio/mean": 0.9876399636268616, "sampling/importance_sampling_ratio/min": 0.6394190192222595, "sampling/sampling_logp_difference/max": 0.5157732963562012, "sampling/sampling_logp_difference/mean": 0.021354639902710915, "step": 156, "step_time": 25.676308020716533 }, { "clip_ratio/high_max": 0.0007681013084948063, "clip_ratio/high_mean": 0.0007681013084948063, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007681013084948063, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1442.0, "completions/mean_length": 697.9833984375, "completions/mean_terminated_length": 651.4310302734375, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.17722446843981743, "epoch": 0.22886297376093295, "frac_reward_zero_std": 0.0, "grad_norm": 0.016081683337688446, "learning_rate": 2e-05, "loss": 0.056921664625406265, "num_tokens": 8561214.0, "reward": 0.5925211906433105, "reward_std": 0.4066368341445923, "rewards/correctness/mean": 0.9333333373069763, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.34081217646598816, "rewards/length_penalty/std": 0.20141668617725372, "sampling/importance_sampling_ratio/max": 1.4456247091293335, "sampling/importance_sampling_ratio/mean": 0.993467390537262, "sampling/importance_sampling_ratio/min": 0.6672379970550537, "sampling/sampling_logp_difference/max": 0.40460848808288574, "sampling/sampling_logp_difference/mean": 0.012250185012817383, "step": 157, "step_time": 25.352831785101444 }, { "clip_ratio/high_max": 0.0002853689814704315, "clip_ratio/high_mean": 0.0002853689814704315, "clip_ratio/low_mean": 0.00025238765495790477, "clip_ratio/low_min": 0.00025238765495790477, "clip_ratio/region_mean": 0.0005377566509802515, "completions/clipped_ratio": 0.40000003576278687, "completions/max_length": 2048.0, "completions/max_terminated_length": 2012.0, "completions/mean_length": 1259.4666748046875, "completions/mean_terminated_length": 733.7777709960938, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.35649363199869794, "epoch": 0.2303206997084548, "frac_reward_zero_std": 0.0, "grad_norm": 0.020887818187475204, "learning_rate": 2e-05, "loss": 0.034592047333717346, "num_tokens": 8641882.0, "reward": -0.03164062649011612, "reward_std": 0.8320719599723816, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.6149739623069763, "rewards/length_penalty/std": 0.35453444719314575, "sampling/importance_sampling_ratio/max": 1.694777011871338, "sampling/importance_sampling_ratio/mean": 0.9877781867980957, "sampling/importance_sampling_ratio/min": 0.5174592733383179, "sampling/sampling_logp_difference/max": 0.6588244438171387, "sampling/sampling_logp_difference/mean": 0.021618111059069633, "step": 158, "step_time": 28.766278768889606 }, { "clip_ratio/high_max": 0.0012234286599171658, "clip_ratio/high_mean": 0.0012234286599171658, "clip_ratio/low_mean": 0.00019811623981998613, "clip_ratio/low_min": 0.00019811623981998613, "clip_ratio/region_mean": 0.0014215449142890673, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1571.0, "completions/mean_length": 800.9166870117188, "completions/mean_terminated_length": 711.8392944335938, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.2685914561152458, "epoch": 0.23177842565597667, "frac_reward_zero_std": 0.0, "grad_norm": 0.015032554976642132, "learning_rate": 2e-05, "loss": 0.04539729654788971, "num_tokens": 8695537.0, "reward": 0.39226076006889343, "reward_std": 0.5275649428367615, "rewards/correctness/mean": 0.7833333611488342, "rewards/correctness/std": 0.41545024514198303, "rewards/length_penalty/mean": -0.3910726010799408, "rewards/length_penalty/std": 0.22598691284656525, "sampling/importance_sampling_ratio/max": 1.5921074151992798, "sampling/importance_sampling_ratio/mean": 0.9907950758934021, "sampling/importance_sampling_ratio/min": 0.6347799897193909, "sampling/sampling_logp_difference/max": 0.4650585651397705, "sampling/sampling_logp_difference/mean": 0.016757389530539513, "step": 159, "step_time": 25.799214388476685 }, { "clip_ratio/high_max": 0.0013589256996056065, "clip_ratio/high_mean": 0.0013589256996056065, "clip_ratio/low_mean": 0.00021470187493832782, "clip_ratio/low_min": 0.00021470187493832782, "clip_ratio/region_mean": 0.001573627606073084, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1981.0, "completions/mean_length": 800.1000366210938, "completions/mean_terminated_length": 686.654541015625, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 0.3904004245996475, "epoch": 0.23323615160349853, "frac_reward_zero_std": 0.0, "grad_norm": 0.019178809598088264, "learning_rate": 2e-05, "loss": 0.042169515043497086, "num_tokens": 8747233.0, "reward": 0.009326172061264515, "reward_std": 0.6670224070549011, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49403220415115356, "rewards/length_penalty/mean": -0.39067381620407104, "rewards/length_penalty/std": 0.2804904282093048, "sampling/importance_sampling_ratio/max": 1.467087745666504, "sampling/importance_sampling_ratio/mean": 0.9868454337120056, "sampling/importance_sampling_ratio/min": 0.5541986227035522, "sampling/sampling_logp_difference/max": 0.5902321338653564, "sampling/sampling_logp_difference/mean": 0.02422356605529785, "step": 160, "step_time": 25.9806922392454 }, { "clip_ratio/high_max": 0.0010678241790931982, "clip_ratio/high_mean": 0.0010678241790931982, "clip_ratio/low_mean": 8.358072348831531e-05, "clip_ratio/low_min": 8.358072348831531e-05, "clip_ratio/region_mean": 0.0011514048965182155, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 761.3333740234375, "completions/mean_terminated_length": 693.614013671875, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.27356397608915967, "epoch": 0.23469387755102042, "frac_reward_zero_std": 0.0, "grad_norm": 0.023295298218727112, "learning_rate": 2e-05, "loss": 0.05298904329538345, "num_tokens": 8797223.0, "reward": 0.44492191076278687, "reward_std": 0.6138378381729126, "rewards/correctness/mean": 0.8166666626930237, "rewards/correctness/std": 0.39020493626594543, "rewards/length_penalty/mean": -0.3717447817325592, "rewards/length_penalty/std": 0.26436156034469604, "sampling/importance_sampling_ratio/max": 1.4474414587020874, "sampling/importance_sampling_ratio/mean": 0.9904724955558777, "sampling/importance_sampling_ratio/min": 0.6545323133468628, "sampling/sampling_logp_difference/max": 0.42383432388305664, "sampling/sampling_logp_difference/mean": 0.01725386641919613, "step": 161, "step_time": 25.533062344184145 }, { "clip_ratio/high_max": 0.0009161523485090584, "clip_ratio/high_mean": 0.0009161523485090584, "clip_ratio/low_mean": 0.00015165507769173323, "clip_ratio/low_min": 0.00015165507769173323, "clip_ratio/region_mean": 0.0010678074031602591, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1835.0, "completions/mean_length": 614.1666870117188, "completions/mean_terminated_length": 589.8643798828125, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.2376063788930575, "epoch": 0.23615160349854228, "frac_reward_zero_std": 0.0, "grad_norm": 0.015938283875584602, "learning_rate": 2e-05, "loss": 0.056312985718250275, "num_tokens": 8837513.0, "reward": 0.48344728350639343, "reward_std": 0.5652655363082886, "rewards/correctness/mean": 0.7833333611488342, "rewards/correctness/std": 0.41545021533966064, "rewards/length_penalty/mean": -0.2998860776424408, "rewards/length_penalty/std": 0.18964459002017975, "sampling/importance_sampling_ratio/max": 1.5897924900054932, "sampling/importance_sampling_ratio/mean": 0.9915858507156372, "sampling/importance_sampling_ratio/min": 0.6721543669700623, "sampling/sampling_logp_difference/max": 0.46360349655151367, "sampling/sampling_logp_difference/mean": 0.015679962933063507, "step": 162, "step_time": 24.472435717703775 }, { "clip_ratio/high_max": 0.0013966952683404088, "clip_ratio/high_mean": 0.0013966952683404088, "clip_ratio/low_mean": 2.8651783698781703e-05, "clip_ratio/low_min": 2.8651783698781703e-05, "clip_ratio/region_mean": 0.001425347038699935, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1896.0, "completions/mean_length": 1025.6500244140625, "completions/mean_terminated_length": 770.0625, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.2782551323374112, "epoch": 0.23760932944606414, "frac_reward_zero_std": 0.0, "grad_norm": 0.014002242125570774, "learning_rate": 2e-05, "loss": 0.022032344713807106, "num_tokens": 8907392.0, "reward": 0.08252767473459244, "reward_std": 0.7483339905738831, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.500805675983429, "rewards/length_penalty/std": 0.3090408444404602, "sampling/importance_sampling_ratio/max": 1.6037006378173828, "sampling/importance_sampling_ratio/mean": 0.9900583028793335, "sampling/importance_sampling_ratio/min": 0.6024936437606812, "sampling/sampling_logp_difference/max": 0.5066781044006348, "sampling/sampling_logp_difference/mean": 0.018255088478326797, "step": 163, "step_time": 28.169299229746684 }, { "clip_ratio/high_max": 0.0012512138249197353, "clip_ratio/high_mean": 0.0012512138249197353, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012512138249197353, "completions/clipped_ratio": 0.0, "completions/max_length": 560.0, "completions/max_terminated_length": 560.0, "completions/mean_length": 384.5166931152344, "completions/mean_terminated_length": 384.5166931152344, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.1906136597196261, "epoch": 0.239067055393586, "frac_reward_zero_std": 0.0, "grad_norm": 0.00936169270426035, "learning_rate": 2e-05, "loss": -0.0005168644711375237, "num_tokens": 8934753.0, "reward": 0.7122477293014526, "reward_std": 0.30102622509002686, "rewards/correctness/mean": 0.8999999761581421, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.18775227665901184, "rewards/length_penalty/std": 0.05292113497853279, "sampling/importance_sampling_ratio/max": 1.4079543352127075, "sampling/importance_sampling_ratio/mean": 0.9925669431686401, "sampling/importance_sampling_ratio/min": 0.6690069437026978, "sampling/sampling_logp_difference/max": 0.4019608497619629, "sampling/sampling_logp_difference/mean": 0.013436808250844479, "step": 164, "step_time": 7.938276870874688 }, { "clip_ratio/high_max": 0.0005091577768325806, "clip_ratio/high_mean": 0.0005091577768325806, "clip_ratio/low_mean": 5.713850987376645e-05, "clip_ratio/low_min": 5.713850987376645e-05, "clip_ratio/region_mean": 0.0005662962891316662, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 2022.0, "completions/mean_length": 882.8500366210938, "completions/mean_terminated_length": 677.2352905273438, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 0.301129013299942, "epoch": 0.24052478134110788, "frac_reward_zero_std": 0.0, "grad_norm": 0.011507508344948292, "learning_rate": 2e-05, "loss": 0.028687074780464172, "num_tokens": 8992344.0, "reward": 0.20225423574447632, "reward_std": 0.7268537282943726, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.43107908964157104, "rewards/length_penalty/std": 0.30351847410202026, "sampling/importance_sampling_ratio/max": 1.4683129787445068, "sampling/importance_sampling_ratio/mean": 0.9888500571250916, "sampling/importance_sampling_ratio/min": 0.6509833335876465, "sampling/sampling_logp_difference/max": 0.42927122116088867, "sampling/sampling_logp_difference/mean": 0.019634412601590157, "step": 165, "step_time": 26.11276886612177 }, { "clip_ratio/high_max": 0.0008709375009251138, "clip_ratio/high_mean": 0.0008709375009251138, "clip_ratio/low_mean": 0.00027229892536221695, "clip_ratio/low_min": 0.00027229892536221695, "clip_ratio/region_mean": 0.0011432364020341386, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1950.0, "completions/mean_length": 935.9000244140625, "completions/mean_terminated_length": 657.875, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.4242909774184227, "epoch": 0.24198250728862974, "frac_reward_zero_std": 0.0, "grad_norm": 0.015190168283879757, "learning_rate": 2e-05, "loss": 0.015511132776737213, "num_tokens": 9051968.0, "reward": 0.29301759600639343, "reward_std": 0.7033945918083191, "rewards/correctness/mean": 0.75, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.45698243379592896, "rewards/length_penalty/std": 0.30968427658081055, "sampling/importance_sampling_ratio/max": 1.5805938243865967, "sampling/importance_sampling_ratio/mean": 0.9843783378601074, "sampling/importance_sampling_ratio/min": 0.6149616241455078, "sampling/sampling_logp_difference/max": 0.48619544506073, "sampling/sampling_logp_difference/mean": 0.027219604700803757, "step": 166, "step_time": 26.639186922926456 }, { "clip_ratio/high_max": 0.0007828866728232242, "clip_ratio/high_mean": 0.0007828866728232242, "clip_ratio/low_mean": 3.87596907482172e-05, "clip_ratio/low_min": 3.87596907482172e-05, "clip_ratio/region_mean": 0.000821646358720803, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 425.5166931152344, "completions/mean_terminated_length": 425.5166931152344, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.1843807970484098, "epoch": 0.2434402332361516, "frac_reward_zero_std": 0.0, "grad_norm": 0.01017924677580595, "learning_rate": 2e-05, "loss": -0.004341884516179562, "num_tokens": 9080519.0, "reward": 0.7422282099723816, "reward_std": 0.21998131275177002, "rewards/correctness/mean": 0.949999988079071, "rewards/correctness/std": 0.2197841852903366, "rewards/length_penalty/mean": -0.20777180790901184, "rewards/length_penalty/std": 0.034709908068180084, "sampling/importance_sampling_ratio/max": 1.5466643571853638, "sampling/importance_sampling_ratio/mean": 0.9935150742530823, "sampling/importance_sampling_ratio/min": 0.6078754663467407, "sampling/sampling_logp_difference/max": 0.49778521060943604, "sampling/sampling_logp_difference/mean": 0.013069001026451588, "step": 167, "step_time": 9.47851428994909 }, { "clip_ratio/high_max": 0.0007199387158228395, "clip_ratio/high_mean": 0.0007199387158228395, "clip_ratio/low_mean": 0.00015790038863390995, "clip_ratio/low_min": 0.00015790038863390995, "clip_ratio/region_mean": 0.000877839093542813, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 2035.0, "completions/mean_length": 713.36669921875, "completions/mean_terminated_length": 592.0363159179688, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.28486377497514087, "epoch": 0.24489795918367346, "frac_reward_zero_std": 0.0, "grad_norm": 0.017850568518042564, "learning_rate": 2e-05, "loss": 0.0237590279430151, "num_tokens": 9128221.0, "reward": 0.45167645812034607, "reward_std": 0.6125007271766663, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.3483235538005829, "rewards/length_penalty/std": 0.2630182206630707, "sampling/importance_sampling_ratio/max": 1.4393036365509033, "sampling/importance_sampling_ratio/mean": 0.9893081784248352, "sampling/importance_sampling_ratio/min": 0.6482274532318115, "sampling/sampling_logp_difference/max": 0.4335136413574219, "sampling/sampling_logp_difference/mean": 0.019400296732783318, "step": 168, "step_time": 25.452282634796575 }, { "clip_ratio/high_max": 0.000811036714973549, "clip_ratio/high_mean": 0.000811036714973549, "clip_ratio/low_mean": 0.0002101373320329003, "clip_ratio/low_min": 0.0002101373320329003, "clip_ratio/region_mean": 0.0010211740445811301, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 2048.0, "completions/mean_length": 1134.5833740234375, "completions/mean_terminated_length": 951.8999633789062, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.38400957981745404, "epoch": 0.24635568513119532, "frac_reward_zero_std": 0.0, "grad_norm": 0.0206043291836977, "learning_rate": 2e-05, "loss": 0.0436839833855629, "num_tokens": 9202736.0, "reward": -0.15399578213691711, "reward_std": 0.6629047989845276, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.5539957880973816, "rewards/length_penalty/std": 0.30165985226631165, "sampling/importance_sampling_ratio/max": 1.6086947917938232, "sampling/importance_sampling_ratio/mean": 0.9861995577812195, "sampling/importance_sampling_ratio/min": 0.6074703335762024, "sampling/sampling_logp_difference/max": 0.49845194816589355, "sampling/sampling_logp_difference/mean": 0.024005858227610588, "step": 169, "step_time": 27.775138321798295 }, { "clip_ratio/high_max": 0.000899404549272731, "clip_ratio/high_mean": 0.000899404549272731, "clip_ratio/low_mean": 0.000148629956432463, "clip_ratio/low_min": 0.000148629956432463, "clip_ratio/region_mean": 0.0010480345032798748, "completions/clipped_ratio": 0.0, "completions/max_length": 1892.0, "completions/max_terminated_length": 1892.0, "completions/mean_length": 623.2166748046875, "completions/mean_terminated_length": 623.2166748046875, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.2634037708242734, "epoch": 0.2478134110787172, "frac_reward_zero_std": 0.0, "grad_norm": 0.010958128608763218, "learning_rate": 2e-05, "loss": -0.003399193286895752, "num_tokens": 9245929.0, "reward": 0.24569499492645264, "reward_std": 0.5535570383071899, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.3043050169944763, "rewards/length_penalty/std": 0.15333330631256104, "sampling/importance_sampling_ratio/max": 1.6163140535354614, "sampling/importance_sampling_ratio/mean": 0.9908414483070374, "sampling/importance_sampling_ratio/min": 0.5295116305351257, "sampling/sampling_logp_difference/max": 0.6358001232147217, "sampling/sampling_logp_difference/mean": 0.017154477536678314, "step": 170, "step_time": 24.2347475157585 }, { "clip_ratio/high_max": 0.0006975887202618954, "clip_ratio/high_mean": 0.0006975887202618954, "clip_ratio/low_mean": 0.00013354327044604966, "clip_ratio/low_min": 0.00013354327044604966, "clip_ratio/region_mean": 0.0008311319931332642, "completions/clipped_ratio": 0.0, "completions/max_length": 859.0, "completions/max_terminated_length": 859.0, "completions/mean_length": 482.7500305175781, "completions/mean_terminated_length": 482.7500305175781, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.2617703974246979, "epoch": 0.24927113702623907, "frac_reward_zero_std": 0.0, "grad_norm": 0.013744283467531204, "learning_rate": 2e-05, "loss": 0.03806378319859505, "num_tokens": 9279434.0, "reward": 0.5309489369392395, "reward_std": 0.45812374353408813, "rewards/correctness/mean": 0.7666666507720947, "rewards/correctness/std": 0.42652183771133423, "rewards/length_penalty/mean": -0.2357177734375, "rewards/length_penalty/std": 0.09047216922044754, "sampling/importance_sampling_ratio/max": 1.6830754280090332, "sampling/importance_sampling_ratio/mean": 0.990826427936554, "sampling/importance_sampling_ratio/min": 0.6258879899978638, "sampling/sampling_logp_difference/max": 0.520622730255127, "sampling/sampling_logp_difference/mean": 0.0171992015093565, "step": 171, "step_time": 11.817887012613937 }, { "clip_ratio/high_max": 0.0007639260002179071, "clip_ratio/high_mean": 0.0007639260002179071, "clip_ratio/low_mean": 3.643783808608229e-05, "clip_ratio/low_min": 3.643783808608229e-05, "clip_ratio/region_mean": 0.0008003638358786702, "completions/clipped_ratio": 0.0, "completions/max_length": 859.0, "completions/max_terminated_length": 859.0, "completions/mean_length": 418.4000244140625, "completions/mean_terminated_length": 418.4000244140625, "completions/min_length": 103.0, "completions/min_terminated_length": 103.0, "entropy": 0.20873272667328516, "epoch": 0.25072886297376096, "frac_reward_zero_std": 0.0, "grad_norm": 0.010405924171209335, "learning_rate": 2e-05, "loss": 0.010939408093690872, "num_tokens": 9308038.0, "reward": 0.7623698115348816, "reward_std": 0.21447253227233887, "rewards/correctness/mean": 0.9666666388511658, "rewards/correctness/std": 0.1810203343629837, "rewards/length_penalty/mean": -0.20429687201976776, "rewards/length_penalty/std": 0.08035586029291153, "sampling/importance_sampling_ratio/max": 1.450929045677185, "sampling/importance_sampling_ratio/mean": 0.9924795031547546, "sampling/importance_sampling_ratio/min": 0.5934280157089233, "sampling/sampling_logp_difference/max": 0.5218393802642822, "sampling/sampling_logp_difference/mean": 0.01417568325996399, "step": 172, "step_time": 10.970934886718169 }, { "clip_ratio/high_max": 0.0006309296659310348, "clip_ratio/high_mean": 0.0006309296659310348, "clip_ratio/low_mean": 5.6718282091120877e-05, "clip_ratio/low_min": 5.6718282091120877e-05, "clip_ratio/region_mean": 0.0006876479480221557, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1610.0, "completions/mean_length": 570.4500122070312, "completions/mean_terminated_length": 519.5, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.2216138318181038, "epoch": 0.2521865889212828, "frac_reward_zero_std": 0.0, "grad_norm": 0.008872742764651775, "learning_rate": 2e-05, "loss": -0.0019448311068117619, "num_tokens": 9346215.0, "reward": 0.2047933042049408, "reward_std": 0.617644727230072, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.2785400450229645, "rewards/length_penalty/std": 0.19476625323295593, "sampling/importance_sampling_ratio/max": 1.6034185886383057, "sampling/importance_sampling_ratio/mean": 0.9922030568122864, "sampling/importance_sampling_ratio/min": 0.6622313261032104, "sampling/sampling_logp_difference/max": 0.4721379280090332, "sampling/sampling_logp_difference/mean": 0.01476544700562954, "step": 173, "step_time": 24.293195291189477 }, { "clip_ratio/high_max": 0.0005930817230061317, "clip_ratio/high_mean": 0.0005930817230061317, "clip_ratio/low_mean": 0.0001003987369282792, "clip_ratio/low_min": 0.0001003987369282792, "clip_ratio/region_mean": 0.0006934804647850493, "completions/clipped_ratio": 0.0, "completions/max_length": 1230.0, "completions/max_terminated_length": 1230.0, "completions/mean_length": 533.1666870117188, "completions/mean_terminated_length": 533.1666870117188, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.1814538538455963, "epoch": 0.2536443148688047, "frac_reward_zero_std": 0.0, "grad_norm": 0.012497815303504467, "learning_rate": 2e-05, "loss": 0.031932175159454346, "num_tokens": 9382015.0, "reward": 0.6729980707168579, "reward_std": 0.3128434121608734, "rewards/correctness/mean": 0.9333333373069763, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.2603352963924408, "rewards/length_penalty/std": 0.111988864839077, "sampling/importance_sampling_ratio/max": 1.600244402885437, "sampling/importance_sampling_ratio/mean": 0.993462085723877, "sampling/importance_sampling_ratio/min": 0.45217838883399963, "sampling/sampling_logp_difference/max": 0.7936785221099854, "sampling/sampling_logp_difference/mean": 0.012692712247371674, "step": 174, "step_time": 15.74416776606813 }, { "clip_ratio/high_max": 0.0005973838512242461, "clip_ratio/high_mean": 0.0005973838512242461, "clip_ratio/low_mean": 0.00025322781584691256, "clip_ratio/low_min": 0.00025322781584691256, "clip_ratio/region_mean": 0.0008506116670711587, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 2044.0, "completions/mean_length": 1029.166748046875, "completions/mean_terminated_length": 774.4583740234375, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.30322639395793277, "epoch": 0.25510204081632654, "frac_reward_zero_std": 0.0, "grad_norm": 0.015394343994557858, "learning_rate": 2e-05, "loss": 0.0004860721528530121, "num_tokens": 9450505.0, "reward": 0.14747722446918488, "reward_std": 0.7596887350082397, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.5025227665901184, "rewards/length_penalty/std": 0.3278689980506897, "sampling/importance_sampling_ratio/max": 1.474064588546753, "sampling/importance_sampling_ratio/mean": 0.9891459345817566, "sampling/importance_sampling_ratio/min": 0.6035506725311279, "sampling/sampling_logp_difference/max": 0.5049252510070801, "sampling/sampling_logp_difference/mean": 0.019591135904192924, "step": 175, "step_time": 27.490358071168885 }, { "clip_ratio/high_max": 0.001002947586433341, "clip_ratio/high_mean": 0.001002947586433341, "clip_ratio/low_mean": 0.00013154309999663383, "clip_ratio/low_min": 0.00013154309999663383, "clip_ratio/region_mean": 0.0011344906912806134, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1703.0, "completions/mean_length": 1014.2500610351562, "completions/mean_terminated_length": 899.388916015625, "completions/min_length": 462.0, "completions/min_terminated_length": 462.0, "entropy": 0.40745416780312854, "epoch": 0.2565597667638484, "frac_reward_zero_std": 0.0, "grad_norm": 0.017374936491250992, "learning_rate": 2e-05, "loss": 0.04549677297472954, "num_tokens": 9517570.0, "reward": -0.09523925930261612, "reward_std": 0.6463657021522522, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.4952392578125, "rewards/length_penalty/std": 0.23023425042629242, "sampling/importance_sampling_ratio/max": 1.4509283304214478, "sampling/importance_sampling_ratio/mean": 0.9854934215545654, "sampling/importance_sampling_ratio/min": 0.47510966658592224, "sampling/sampling_logp_difference/max": 0.7442096471786499, "sampling/sampling_logp_difference/mean": 0.024702923372387886, "step": 176, "step_time": 27.569041184149683 }, { "clip_ratio/high_max": 0.0009653393450813988, "clip_ratio/high_mean": 0.0009653393450813988, "clip_ratio/low_mean": 8.393301201673846e-05, "clip_ratio/low_min": 8.393301201673846e-05, "clip_ratio/region_mean": 0.0010492723570981373, "completions/clipped_ratio": 0.0, "completions/max_length": 1833.0, "completions/max_terminated_length": 1833.0, "completions/mean_length": 585.8333740234375, "completions/mean_terminated_length": 585.8333740234375, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.314935398598512, "epoch": 0.25801749271137026, "frac_reward_zero_std": 0.0, "grad_norm": 0.01463930681347847, "learning_rate": 2e-05, "loss": -0.006962409242987633, "num_tokens": 9557790.0, "reward": 0.3139485716819763, "reward_std": 0.5434808731079102, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.2860514223575592, "rewards/length_penalty/std": 0.15339350700378418, "sampling/importance_sampling_ratio/max": 1.5201925039291382, "sampling/importance_sampling_ratio/mean": 0.9881151914596558, "sampling/importance_sampling_ratio/min": 0.6471298336982727, "sampling/sampling_logp_difference/max": 0.4352083206176758, "sampling/sampling_logp_difference/mean": 0.021870296448469162, "step": 177, "step_time": 22.817483997903764 }, { "clip_ratio/high_max": 0.001856392181556051, "clip_ratio/high_mean": 0.001856392181556051, "clip_ratio/low_mean": 0.00014501971600111574, "clip_ratio/low_min": 0.00014501971600111574, "clip_ratio/region_mean": 0.002001411863602698, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1962.0, "completions/mean_length": 705.0167236328125, "completions/mean_terminated_length": 682.2542114257812, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.2905154253045718, "epoch": 0.2594752186588921, "frac_reward_zero_std": 0.0, "grad_norm": 0.015073919668793678, "learning_rate": 2e-05, "loss": 0.023865118622779846, "num_tokens": 9604961.0, "reward": 0.3724202811717987, "reward_std": 0.5507521629333496, "rewards/correctness/mean": 0.7166666388511658, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.34424641728401184, "rewards/length_penalty/std": 0.1690424531698227, "sampling/importance_sampling_ratio/max": 1.9097141027450562, "sampling/importance_sampling_ratio/mean": 0.9898587465286255, "sampling/importance_sampling_ratio/min": 0.6631399393081665, "sampling/sampling_logp_difference/max": 0.6469535827636719, "sampling/sampling_logp_difference/mean": 0.018903009593486786, "step": 178, "step_time": 25.403236869955435 }, { "clip_ratio/high_max": 0.0015735746322510142, "clip_ratio/high_mean": 0.0015735746322510142, "clip_ratio/low_mean": 0.00014540493915167949, "clip_ratio/low_min": 0.00014540493915167949, "clip_ratio/region_mean": 0.0017189795811039705, "completions/clipped_ratio": 0.0, "completions/max_length": 2020.0, "completions/max_terminated_length": 2020.0, "completions/mean_length": 600.7833862304688, "completions/mean_terminated_length": 600.7833862304688, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 0.26812274505694705, "epoch": 0.260932944606414, "frac_reward_zero_std": 0.0, "grad_norm": 0.013311340473592281, "learning_rate": 2e-05, "loss": 0.021306850016117096, "num_tokens": 9645838.0, "reward": 0.30664879083633423, "reward_std": 0.5342420339584351, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.2933512330055237, "rewards/length_penalty/std": 0.188986137509346, "sampling/importance_sampling_ratio/max": 1.5045559406280518, "sampling/importance_sampling_ratio/mean": 0.9907915592193604, "sampling/importance_sampling_ratio/min": 0.6281037330627441, "sampling/sampling_logp_difference/max": 0.46504998207092285, "sampling/sampling_logp_difference/mean": 0.01711450330913067, "step": 179, "step_time": 24.340147527866066 }, { "clip_ratio/high_max": 0.000845977726082007, "clip_ratio/high_mean": 0.000845977726082007, "clip_ratio/low_mean": 0.0001156907164840959, "clip_ratio/low_min": 0.0001156907164840959, "clip_ratio/region_mean": 0.0009616684498420606, "completions/clipped_ratio": 0.0, "completions/max_length": 1717.0, "completions/max_terminated_length": 1717.0, "completions/mean_length": 743.8500366210938, "completions/mean_terminated_length": 743.8500366210938, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.28752462814251584, "epoch": 0.26239067055393583, "frac_reward_zero_std": 0.0, "grad_norm": 0.012962594628334045, "learning_rate": 2e-05, "loss": 0.0020775552839040756, "num_tokens": 9694649.0, "reward": 0.1701253354549408, "reward_std": 0.5588796734809875, "rewards/correctness/mean": 0.5333333611488342, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.36320799589157104, "rewards/length_penalty/std": 0.15787248313426971, "sampling/importance_sampling_ratio/max": 1.4653091430664062, "sampling/importance_sampling_ratio/mean": 0.9898093342781067, "sampling/importance_sampling_ratio/min": 0.6383785009384155, "sampling/sampling_logp_difference/max": 0.4488239288330078, "sampling/sampling_logp_difference/mean": 0.018891487270593643, "step": 180, "step_time": 22.350864383857697 }, { "clip_ratio/high_max": 0.001120295113651082, "clip_ratio/high_mean": 0.001120295113651082, "clip_ratio/low_mean": 6.471350692057361e-05, "clip_ratio/low_min": 6.471350692057361e-05, "clip_ratio/region_mean": 0.0011850086060197402, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1806.0, "completions/mean_length": 878.3833618164062, "completions/mean_terminated_length": 816.8245849609375, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.2872309908270836, "epoch": 0.26384839650145775, "frac_reward_zero_std": 0.0, "grad_norm": 0.0205439031124115, "learning_rate": 2e-05, "loss": 0.06915633380413055, "num_tokens": 9751932.0, "reward": 0.4211018979549408, "reward_std": 0.5352883338928223, "rewards/correctness/mean": 0.8500000238418579, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.4288981258869171, "rewards/length_penalty/std": 0.23621498048305511, "sampling/importance_sampling_ratio/max": 1.4063832759857178, "sampling/importance_sampling_ratio/mean": 0.9895955324172974, "sampling/importance_sampling_ratio/min": 0.6374525427818298, "sampling/sampling_logp_difference/max": 0.4502754211425781, "sampling/sampling_logp_difference/mean": 0.018434900790452957, "step": 181, "step_time": 25.942147817928344 }, { "clip_ratio/high_max": 0.0006055711128283292, "clip_ratio/high_mean": 0.0006055711128283292, "clip_ratio/low_mean": 0.00015449593047378585, "clip_ratio/low_min": 0.00015449593047378585, "clip_ratio/region_mean": 0.0007600670360261574, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 834.0, "completions/mean_length": 663.0167236328125, "completions/mean_terminated_length": 537.1090698242188, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.2742292533318202, "epoch": 0.2653061224489796, "frac_reward_zero_std": 0.0, "grad_norm": 0.015478339046239853, "learning_rate": 2e-05, "loss": 0.0617741234600544, "num_tokens": 9796143.0, "reward": 0.32626140117645264, "reward_std": 0.6145201921463013, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.32373860478401184, "rewards/length_penalty/std": 0.21554671227931976, "sampling/importance_sampling_ratio/max": 1.8416732549667358, "sampling/importance_sampling_ratio/mean": 0.9902825355529785, "sampling/importance_sampling_ratio/min": 0.4849720895290375, "sampling/sampling_logp_difference/max": 0.7236639261245728, "sampling/sampling_logp_difference/mean": 0.01882294937968254, "step": 182, "step_time": 24.860169053077698 }, { "clip_ratio/high_max": 0.0012968556402483955, "clip_ratio/high_mean": 0.0012968556402483955, "clip_ratio/low_mean": 0.00016336666885763407, "clip_ratio/low_min": 0.00016336666885763407, "clip_ratio/region_mean": 0.0014602223091060296, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1839.0, "completions/mean_length": 564.8500366210938, "completions/mean_terminated_length": 539.7118530273438, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.3727461298306783, "epoch": 0.26676384839650147, "frac_reward_zero_std": 0.0, "grad_norm": 0.01920958235859871, "learning_rate": 2e-05, "loss": 0.033165499567985535, "num_tokens": 9833634.0, "reward": 0.14086100459098816, "reward_std": 0.5286503434181213, "rewards/correctness/mean": 0.4166666567325592, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.27580565214157104, "rewards/length_penalty/std": 0.18563447892665863, "sampling/importance_sampling_ratio/max": 1.4753550291061401, "sampling/importance_sampling_ratio/mean": 0.9871658682823181, "sampling/importance_sampling_ratio/min": 0.6304611563682556, "sampling/sampling_logp_difference/max": 0.4613037109375, "sampling/sampling_logp_difference/mean": 0.022531885653734207, "step": 183, "step_time": 24.735460065072402 }, { "clip_ratio/high_max": 0.0008379968882460768, "clip_ratio/high_mean": 0.0008379968882460768, "clip_ratio/low_mean": 6.425392105787371e-05, "clip_ratio/low_min": 6.425392105787371e-05, "clip_ratio/region_mean": 0.0009022508165799081, "completions/clipped_ratio": 0.0, "completions/max_length": 1879.0, "completions/max_terminated_length": 1879.0, "completions/mean_length": 744.4667358398438, "completions/mean_terminated_length": 744.4667358398438, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "entropy": 0.1703206648429235, "epoch": 0.26822157434402333, "frac_reward_zero_std": 0.0, "grad_norm": 0.01721379905939102, "learning_rate": 2e-05, "loss": -0.0008378904312849045, "num_tokens": 9883722.0, "reward": 0.36982423067092896, "reward_std": 0.4271937906742096, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.3635091185569763, "rewards/length_penalty/std": 0.2257184088230133, "sampling/importance_sampling_ratio/max": 1.4273576736450195, "sampling/importance_sampling_ratio/mean": 0.9935067892074585, "sampling/importance_sampling_ratio/min": 0.4246005117893219, "sampling/sampling_logp_difference/max": 0.8566066026687622, "sampling/sampling_logp_difference/mean": 0.012150940485298634, "step": 184, "step_time": 23.781651353929192 }, { "clip_ratio/high_max": 0.000999829791301939, "clip_ratio/high_mean": 0.000999829791301939, "clip_ratio/low_mean": 0.0002239340950230447, "clip_ratio/low_min": 0.0002239340950230447, "clip_ratio/region_mean": 0.0012237638802616857, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1923.0, "completions/mean_length": 921.5833740234375, "completions/mean_terminated_length": 748.2885131835938, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.4105442514022191, "epoch": 0.2696793002915452, "frac_reward_zero_std": 0.0, "grad_norm": 0.01904107816517353, "learning_rate": 2e-05, "loss": 0.10748829692602158, "num_tokens": 9944517.0, "reward": 0.11667481064796448, "reward_std": 0.6949194073677063, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971747398376465, "rewards/length_penalty/mean": -0.4499918520450592, "rewards/length_penalty/std": 0.2978811264038086, "sampling/importance_sampling_ratio/max": 1.4936118125915527, "sampling/importance_sampling_ratio/mean": 0.9865139722824097, "sampling/importance_sampling_ratio/min": 0.47563496232032776, "sampling/sampling_logp_difference/max": 0.7431045770645142, "sampling/sampling_logp_difference/mean": 0.024014528840780258, "step": 185, "step_time": 26.656971606891602 }, { "clip_ratio/high_max": 0.00039769137583789416, "clip_ratio/high_mean": 0.00039769137583789416, "clip_ratio/low_mean": 0.0002477293007056384, "clip_ratio/low_min": 0.0002477293007056384, "clip_ratio/region_mean": 0.0006454206741182134, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1886.0, "completions/mean_length": 832.6000366210938, "completions/mean_terminated_length": 645.6154174804688, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.21153807391722998, "epoch": 0.27113702623906705, "frac_reward_zero_std": 0.0, "grad_norm": 0.01534296479076147, "learning_rate": 2e-05, "loss": 0.034561239182949066, "num_tokens": 9998603.0, "reward": 0.11012370139360428, "reward_std": 0.7228626608848572, "rewards/correctness/mean": 0.5166666507720947, "rewards/correctness/std": 0.5039393305778503, "rewards/length_penalty/mean": -0.40654295682907104, "rewards/length_penalty/std": 0.29053258895874023, "sampling/importance_sampling_ratio/max": 1.6479967832565308, "sampling/importance_sampling_ratio/mean": 0.991916835308075, "sampling/importance_sampling_ratio/min": 0.520044207572937, "sampling/sampling_logp_difference/max": 0.653841495513916, "sampling/sampling_logp_difference/mean": 0.014898436143994331, "step": 186, "step_time": 27.10042130202055 }, { "clip_ratio/high_max": 0.000987601451925002, "clip_ratio/high_mean": 0.000987601451925002, "clip_ratio/low_mean": 0.0003099991808994673, "clip_ratio/low_min": 0.0003099991808994673, "clip_ratio/region_mean": 0.0012976006158472349, "completions/clipped_ratio": 0.38333335518836975, "completions/max_length": 2048.0, "completions/max_terminated_length": 1950.0, "completions/mean_length": 1271.166748046875, "completions/mean_terminated_length": 788.270263671875, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.5849319398403168, "epoch": 0.2725947521865889, "frac_reward_zero_std": 0.0, "grad_norm": 0.02560403384268284, "learning_rate": 2e-05, "loss": 0.044088806957006454, "num_tokens": 10081043.0, "reward": -0.10402018576860428, "reward_std": 0.8494263291358948, "rewards/correctness/mean": 0.5166666507720947, "rewards/correctness/std": 0.5039393305778503, "rewards/length_penalty/mean": -0.6206868290901184, "rewards/length_penalty/std": 0.3692076802253723, "sampling/importance_sampling_ratio/max": 1.7236531972885132, "sampling/importance_sampling_ratio/mean": 0.980085551738739, "sampling/importance_sampling_ratio/min": 0.6232811212539673, "sampling/sampling_logp_difference/max": 0.5444459915161133, "sampling/sampling_logp_difference/mean": 0.034810230135917664, "step": 187, "step_time": 28.50091272802092 }, { "clip_ratio/high_max": 0.0009432584435368577, "clip_ratio/high_mean": 0.0009432584435368577, "clip_ratio/low_mean": 0.0001643827126827091, "clip_ratio/low_min": 0.0001643827126827091, "clip_ratio/region_mean": 0.00110764114651829, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1876.0, "completions/mean_length": 950.4334106445312, "completions/mean_terminated_length": 781.5769653320312, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.37916910151640576, "epoch": 0.27405247813411077, "frac_reward_zero_std": 0.0, "grad_norm": 0.017173759639263153, "learning_rate": 2e-05, "loss": 0.016559267416596413, "num_tokens": 10142859.0, "reward": 0.03592122718691826, "reward_std": 0.6352493166923523, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.46407878398895264, "rewards/length_penalty/std": 0.2751929461956024, "sampling/importance_sampling_ratio/max": 1.5932568311691284, "sampling/importance_sampling_ratio/mean": 0.98691725730896, "sampling/importance_sampling_ratio/min": 0.5883161425590515, "sampling/sampling_logp_difference/max": 0.5304908752441406, "sampling/sampling_logp_difference/mean": 0.023954246193170547, "step": 188, "step_time": 27.15778080979362 }, { "clip_ratio/high_max": 0.0006955851373883585, "clip_ratio/high_mean": 0.0006955851373883585, "clip_ratio/low_mean": 0.00011439093577791937, "clip_ratio/low_min": 0.00011439093577791937, "clip_ratio/region_mean": 0.0008099760743789375, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1836.0, "completions/mean_length": 923.300048828125, "completions/mean_terminated_length": 798.3333129882812, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.2857773353656133, "epoch": 0.2755102040816326, "frac_reward_zero_std": 0.0, "grad_norm": 0.01848120614886284, "learning_rate": 2e-05, "loss": 0.030527060851454735, "num_tokens": 10202417.0, "reward": 0.2325032651424408, "reward_std": 0.6638219952583313, "rewards/correctness/mean": 0.6833333373069763, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.4508300721645355, "rewards/length_penalty/std": 0.2595711648464203, "sampling/importance_sampling_ratio/max": 1.5372258424758911, "sampling/importance_sampling_ratio/mean": 0.9894238710403442, "sampling/importance_sampling_ratio/min": 0.6098623275756836, "sampling/sampling_logp_difference/max": 0.4945220947265625, "sampling/sampling_logp_difference/mean": 0.01942998357117176, "step": 189, "step_time": 26.307919850572944 }, { "clip_ratio/high_max": 0.001184566431523611, "clip_ratio/high_mean": 0.001184566431523611, "clip_ratio/low_mean": 0.00018724674737313762, "clip_ratio/low_min": 0.00018724674737313762, "clip_ratio/region_mean": 0.001371813181322068, "completions/clipped_ratio": 0.18333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 1692.0, "completions/mean_length": 1109.25, "completions/mean_terminated_length": 898.5101928710938, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.5130080531040827, "epoch": 0.27696793002915454, "frac_reward_zero_std": 0.0, "grad_norm": 0.018907280638813972, "learning_rate": 2e-05, "loss": 0.03437569737434387, "num_tokens": 10274082.0, "reward": -0.05829264596104622, "reward_std": 0.6425654292106628, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.5416259765625, "rewards/length_penalty/std": 0.2773684859275818, "sampling/importance_sampling_ratio/max": 1.6074111461639404, "sampling/importance_sampling_ratio/mean": 0.9823446869850159, "sampling/importance_sampling_ratio/min": 0.6308839917182922, "sampling/sampling_logp_difference/max": 0.4746248722076416, "sampling/sampling_logp_difference/mean": 0.031214483082294464, "step": 190, "step_time": 27.215038079069927 }, { "clip_ratio/high_max": 0.0009124269078408057, "clip_ratio/high_mean": 0.0009124269078408057, "clip_ratio/low_mean": 0.00021373673977601962, "clip_ratio/low_min": 0.00021373673977601962, "clip_ratio/region_mean": 0.0011261636197256546, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1815.0, "completions/mean_length": 842.4000244140625, "completions/mean_terminated_length": 629.6470947265625, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.42648114264011383, "epoch": 0.2784256559766764, "frac_reward_zero_std": 0.0, "grad_norm": 0.017047202214598656, "learning_rate": 2e-05, "loss": 0.005941006354987621, "num_tokens": 10329286.0, "reward": 0.17200522124767303, "reward_std": 0.6729675531387329, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.41132813692092896, "rewards/length_penalty/std": 0.28831717371940613, "sampling/importance_sampling_ratio/max": 1.7022995948791504, "sampling/importance_sampling_ratio/mean": 0.9851097464561462, "sampling/importance_sampling_ratio/min": 0.6182209253311157, "sampling/sampling_logp_difference/max": 0.531980037689209, "sampling/sampling_logp_difference/mean": 0.027465863153338432, "step": 191, "step_time": 26.72552009532228 }, { "clip_ratio/high_max": 0.001399731185908119, "clip_ratio/high_mean": 0.001399731185908119, "clip_ratio/low_mean": 7.817385873446862e-05, "clip_ratio/low_min": 7.817385873446862e-05, "clip_ratio/region_mean": 0.0014779050446425874, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1720.0, "completions/mean_length": 572.0667114257812, "completions/mean_terminated_length": 547.0508422851562, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.33766258011261624, "epoch": 0.27988338192419826, "frac_reward_zero_std": 0.0, "grad_norm": 0.023948965594172478, "learning_rate": 2e-05, "loss": 0.06839627027511597, "num_tokens": 10369010.0, "reward": 0.2873372435569763, "reward_std": 0.5584322214126587, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.27932941913604736, "rewards/length_penalty/std": 0.18996958434581757, "sampling/importance_sampling_ratio/max": 1.6533702611923218, "sampling/importance_sampling_ratio/mean": 0.9873671531677246, "sampling/importance_sampling_ratio/min": 0.5788065195083618, "sampling/sampling_logp_difference/max": 0.5467870235443115, "sampling/sampling_logp_difference/mean": 0.02334241196513176, "step": 192, "step_time": 24.896456439048052 }, { "clip_ratio/high_max": 0.0007610242513086026, "clip_ratio/high_mean": 0.0007610242513086026, "clip_ratio/low_mean": 0.0002833080046305743, "clip_ratio/low_min": 0.0002833080046305743, "clip_ratio/region_mean": 0.0010443322389619425, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 2048.0, "completions/mean_length": 871.2000732421875, "completions/mean_terminated_length": 740.4444580078125, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.3991595556338628, "epoch": 0.2813411078717201, "frac_reward_zero_std": 0.0, "grad_norm": 0.022390838712453842, "learning_rate": 2e-05, "loss": 0.06970308721065521, "num_tokens": 10427492.0, "reward": 0.04127604514360428, "reward_std": 0.6397515535354614, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.4253906309604645, "rewards/length_penalty/std": 0.3054787814617157, "sampling/importance_sampling_ratio/max": 1.798714518547058, "sampling/importance_sampling_ratio/mean": 0.9861185550689697, "sampling/importance_sampling_ratio/min": 0.5218219757080078, "sampling/sampling_logp_difference/max": 0.6504287719726562, "sampling/sampling_logp_difference/mean": 0.025752553716301918, "step": 193, "step_time": 27.298154526157305 }, { "clip_ratio/high_max": 0.0005429390585049987, "clip_ratio/high_mean": 0.0005429390585049987, "clip_ratio/low_mean": 0.00016068309923866764, "clip_ratio/low_min": 0.00016068309923866764, "clip_ratio/region_mean": 0.0007036221407664319, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1571.0, "completions/mean_length": 465.9833679199219, "completions/mean_terminated_length": 439.16949462890625, "completions/min_length": 112.0, "completions/min_terminated_length": 112.0, "entropy": 0.25524456550677616, "epoch": 0.282798833819242, "frac_reward_zero_std": 0.0, "grad_norm": 0.018759645521640778, "learning_rate": 2e-05, "loss": 0.010321836918592453, "num_tokens": 10459081.0, "reward": 0.6891357898712158, "reward_std": 0.3848750591278076, "rewards/correctness/mean": 0.9166666865348816, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.22753092646598816, "rewards/length_penalty/std": 0.15539667010307312, "sampling/importance_sampling_ratio/max": 1.6560379266738892, "sampling/importance_sampling_ratio/mean": 0.990645170211792, "sampling/importance_sampling_ratio/min": 0.6468564867973328, "sampling/sampling_logp_difference/max": 0.5044279098510742, "sampling/sampling_logp_difference/mean": 0.018840471282601357, "step": 194, "step_time": 23.62958171311766 }, { "clip_ratio/high_max": 0.0015640522178728133, "clip_ratio/high_mean": 0.0015640522178728133, "clip_ratio/low_mean": 0.00019902208684167513, "clip_ratio/low_min": 0.00019902208684167513, "clip_ratio/region_mean": 0.0017630742610587429, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1778.0, "completions/mean_length": 976.2333984375, "completions/mean_terminated_length": 811.34619140625, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.44658708075682324, "epoch": 0.28425655976676384, "frac_reward_zero_std": 0.0, "grad_norm": 0.021446384489536285, "learning_rate": 2e-05, "loss": 0.05115346610546112, "num_tokens": 10522335.0, "reward": 0.22332358360290527, "reward_std": 0.670845091342926, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.47667643427848816, "rewards/length_penalty/std": 0.2933204472064972, "sampling/importance_sampling_ratio/max": 1.6302381753921509, "sampling/importance_sampling_ratio/mean": 0.9849187731742859, "sampling/importance_sampling_ratio/min": 0.5913414359092712, "sampling/sampling_logp_difference/max": 0.5253617763519287, "sampling/sampling_logp_difference/mean": 0.027165725827217102, "step": 195, "step_time": 27.863610920263454 }, { "clip_ratio/high_max": 0.0016187613170283537, "clip_ratio/high_mean": 0.0016187613170283537, "clip_ratio/low_mean": 0.00023703551172123602, "clip_ratio/low_min": 0.00023703551172123602, "clip_ratio/region_mean": 0.0018557968433015049, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 2008.0, "completions/mean_length": 949.6500244140625, "completions/mean_terminated_length": 849.7999877929688, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.4802217831214269, "epoch": 0.2857142857142857, "frac_reward_zero_std": 0.0, "grad_norm": 0.023930514231324196, "learning_rate": 2e-05, "loss": 0.08557190001010895, "num_tokens": 10583064.0, "reward": 0.30297037959098816, "reward_std": 0.6304841637611389, "rewards/correctness/mean": 0.7666666507720947, "rewards/correctness/std": 0.42652183771133423, "rewards/length_penalty/mean": -0.46369630098342896, "rewards/length_penalty/std": 0.27533650398254395, "sampling/importance_sampling_ratio/max": 1.991895079612732, "sampling/importance_sampling_ratio/mean": 0.9830731153488159, "sampling/importance_sampling_ratio/min": 0.568490207195282, "sampling/sampling_logp_difference/max": 0.6890864372253418, "sampling/sampling_logp_difference/mean": 0.03178530931472778, "step": 196, "step_time": 26.212359979748726 }, { "clip_ratio/high_max": 0.000976633057386304, "clip_ratio/high_mean": 0.000976633057386304, "clip_ratio/low_mean": 0.0002062602531320105, "clip_ratio/low_min": 0.0002062602531320105, "clip_ratio/region_mean": 0.0011828933202195913, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 2022.0, "completions/mean_length": 1065.0167236328125, "completions/mean_terminated_length": 707.5681762695312, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 0.45353777209917706, "epoch": 0.28717201166180756, "frac_reward_zero_std": 0.0, "grad_norm": 0.01922941952943802, "learning_rate": 2e-05, "loss": 0.06035600230097771, "num_tokens": 10654885.0, "reward": 0.04663899913430214, "reward_std": 0.7512121200561523, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.5200276970863342, "rewards/length_penalty/std": 0.3408592641353607, "sampling/importance_sampling_ratio/max": 1.76103937625885, "sampling/importance_sampling_ratio/mean": 0.9833039045333862, "sampling/importance_sampling_ratio/min": 0.5540443062782288, "sampling/sampling_logp_difference/max": 0.5905106067657471, "sampling/sampling_logp_difference/mean": 0.03049941174685955, "step": 197, "step_time": 29.642499959561974 }, { "clip_ratio/high_max": 0.0006672914654094105, "clip_ratio/high_mean": 0.0006672914654094105, "clip_ratio/low_mean": 0.00011224134747559826, "clip_ratio/low_min": 0.00011224134747559826, "clip_ratio/region_mean": 0.0007795328128850088, "completions/clipped_ratio": 0.0, "completions/max_length": 1777.0, "completions/max_terminated_length": 1777.0, "completions/mean_length": 620.1000366210938, "completions/mean_terminated_length": 620.1000366210938, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.22741481413443884, "epoch": 0.2886297376093295, "frac_reward_zero_std": 0.0, "grad_norm": 0.01049730647355318, "learning_rate": 2e-05, "loss": -0.0004327157512307167, "num_tokens": 10696041.0, "reward": 0.4138834774494171, "reward_std": 0.42144575715065, "rewards/correctness/mean": 0.7166666388511658, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.30278319120407104, "rewards/length_penalty/std": 0.2068432718515396, "sampling/importance_sampling_ratio/max": 1.6680134534835815, "sampling/importance_sampling_ratio/mean": 0.9916688799858093, "sampling/importance_sampling_ratio/min": 0.6686152815818787, "sampling/sampling_logp_difference/max": 0.5116333961486816, "sampling/sampling_logp_difference/mean": 0.016084833070635796, "step": 198, "step_time": 21.738332585897297 }, { "clip_ratio/high_max": 0.0010048883559647948, "clip_ratio/high_mean": 0.0010048883559647948, "clip_ratio/low_mean": 0.00013046783957785615, "clip_ratio/low_min": 0.00013046783957785615, "clip_ratio/region_mean": 0.0011353562003932893, "completions/clipped_ratio": 0.0, "completions/max_length": 1556.0, "completions/max_terminated_length": 1556.0, "completions/mean_length": 610.3500366210938, "completions/mean_terminated_length": 610.3500366210938, "completions/min_length": 99.0, "completions/min_terminated_length": 99.0, "entropy": 0.23824912309646606, "epoch": 0.29008746355685133, "frac_reward_zero_std": 0.0, "grad_norm": 0.011849445290863514, "learning_rate": 2e-05, "loss": 0.004184707999229431, "num_tokens": 10737212.0, "reward": 0.36864423751831055, "reward_std": 0.5543534755706787, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.29802244901657104, "rewards/length_penalty/std": 0.16635596752166748, "sampling/importance_sampling_ratio/max": 1.4651864767074585, "sampling/importance_sampling_ratio/mean": 0.9918603301048279, "sampling/importance_sampling_ratio/min": 0.6047338843345642, "sampling/sampling_logp_difference/max": 0.5029667615890503, "sampling/sampling_logp_difference/mean": 0.016273291781544685, "step": 199, "step_time": 19.38298167823814 }, { "clip_ratio/high_max": 0.001253287142996366, "clip_ratio/high_mean": 0.001253287142996366, "clip_ratio/low_mean": 0.00022714178824874884, "clip_ratio/low_min": 0.00022714178824874884, "clip_ratio/region_mean": 0.0014804289482223492, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1888.0, "completions/mean_length": 940.7167358398438, "completions/mean_terminated_length": 817.6851806640625, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.35443704823652905, "epoch": 0.2915451895043732, "frac_reward_zero_std": 0.0, "grad_norm": 0.02682001329958439, "learning_rate": 2e-05, "loss": 0.06590258330106735, "num_tokens": 10798505.0, "reward": 0.07399902492761612, "reward_std": 0.6956524848937988, "rewards/correctness/mean": 0.5333333611488342, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.4593343138694763, "rewards/length_penalty/std": 0.25864270329475403, "sampling/importance_sampling_ratio/max": 1.6371389627456665, "sampling/importance_sampling_ratio/mean": 0.9873014092445374, "sampling/importance_sampling_ratio/min": 0.6002986431121826, "sampling/sampling_logp_difference/max": 0.5103280544281006, "sampling/sampling_logp_difference/mean": 0.02298104204237461, "step": 200, "step_time": 26.898668461013585 }, { "clip_ratio/high_max": 0.0009682189217225338, "clip_ratio/high_mean": 0.0009682189217225338, "clip_ratio/low_mean": 0.00021286548629480725, "clip_ratio/low_min": 0.00021286548629480725, "clip_ratio/region_mean": 0.001181084420143937, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 2022.0, "completions/mean_length": 954.9166870117188, "completions/mean_terminated_length": 810.5471801757812, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "entropy": 0.3482980603973071, "epoch": 0.29300291545189505, "frac_reward_zero_std": 0.0, "grad_norm": 0.017152519896626472, "learning_rate": 2e-05, "loss": 0.007222160696983337, "num_tokens": 10860950.0, "reward": 0.2003987729549408, "reward_std": 0.6431335210800171, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.4662679135799408, "rewards/length_penalty/std": 0.2587616741657257, "sampling/importance_sampling_ratio/max": 1.7061091661453247, "sampling/importance_sampling_ratio/mean": 0.9870877265930176, "sampling/importance_sampling_ratio/min": 0.4480113387107849, "sampling/sampling_logp_difference/max": 0.8029367923736572, "sampling/sampling_logp_difference/mean": 0.023739872500300407, "step": 201, "step_time": 26.416178465355188 }, { "clip_ratio/high_max": 0.002215275550649191, "clip_ratio/high_mean": 0.002215275550649191, "clip_ratio/low_mean": 0.00014715928409714252, "clip_ratio/low_min": 0.00014715928409714252, "clip_ratio/region_mean": 0.002362434839596972, "completions/clipped_ratio": 0.0, "completions/max_length": 1417.0, "completions/max_terminated_length": 1417.0, "completions/mean_length": 580.933349609375, "completions/mean_terminated_length": 580.933349609375, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.39078181982040405, "epoch": 0.2944606413994169, "frac_reward_zero_std": 0.0, "grad_norm": 0.021677112206816673, "learning_rate": 2e-05, "loss": 0.007553763687610626, "num_tokens": 10901066.0, "reward": 0.5996745228767395, "reward_std": 0.39894959330558777, "rewards/correctness/mean": 0.8833333253860474, "rewards/correctness/std": 0.32373178005218506, "rewards/length_penalty/mean": -0.28365886211395264, "rewards/length_penalty/std": 0.1536681354045868, "sampling/importance_sampling_ratio/max": 1.6190487146377563, "sampling/importance_sampling_ratio/mean": 0.9862749576568604, "sampling/importance_sampling_ratio/min": 0.616158127784729, "sampling/sampling_logp_difference/max": 0.48425161838531494, "sampling/sampling_logp_difference/mean": 0.026701323688030243, "step": 202, "step_time": 17.789458194049075 }, { "clip_ratio/high_max": 0.0014989288562598329, "clip_ratio/high_mean": 0.0014989288562598329, "clip_ratio/low_mean": 9.063893715695788e-05, "clip_ratio/low_min": 9.063893715695788e-05, "clip_ratio/region_mean": 0.0015895678273712595, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1475.0, "completions/mean_length": 539.0833740234375, "completions/mean_terminated_length": 513.5084838867188, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.31458817422389984, "epoch": 0.29591836734693877, "frac_reward_zero_std": 0.0, "grad_norm": 0.011786672286689281, "learning_rate": 2e-05, "loss": 0.0017357384786009789, "num_tokens": 10938191.0, "reward": 0.3201090693473816, "reward_std": 0.5758827328681946, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.2632242739200592, "rewards/length_penalty/std": 0.15304800868034363, "sampling/importance_sampling_ratio/max": 1.6922353506088257, "sampling/importance_sampling_ratio/mean": 0.9890303611755371, "sampling/importance_sampling_ratio/min": 0.5799805521965027, "sampling/sampling_logp_difference/max": 0.5447607040405273, "sampling/sampling_logp_difference/mean": 0.021225424483418465, "step": 203, "step_time": 23.974959311774 }, { "clip_ratio/high_max": 0.0005296146603844439, "clip_ratio/high_mean": 0.0005296146603844439, "clip_ratio/low_mean": 0.00010681258208933286, "clip_ratio/low_min": 0.00010681258208933286, "clip_ratio/region_mean": 0.0006364272412611172, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1390.0, "completions/mean_length": 832.0333862304688, "completions/mean_terminated_length": 617.4509887695312, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.30546244978904724, "epoch": 0.29737609329446063, "frac_reward_zero_std": 0.0, "grad_norm": 0.012475666590034962, "learning_rate": 2e-05, "loss": 0.017777489498257637, "num_tokens": 10992363.0, "reward": 0.3937337398529053, "reward_std": 0.633101224899292, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.4062662720680237, "rewards/length_penalty/std": 0.2764950096607208, "sampling/importance_sampling_ratio/max": 1.7180402278900146, "sampling/importance_sampling_ratio/mean": 0.9893898963928223, "sampling/importance_sampling_ratio/min": 0.6034330725669861, "sampling/sampling_logp_difference/max": 0.5411841869354248, "sampling/sampling_logp_difference/mean": 0.02022479847073555, "step": 204, "step_time": 26.32237398601137 }, { "clip_ratio/high_max": 0.0008661264049199721, "clip_ratio/high_mean": 0.0008661264049199721, "clip_ratio/low_mean": 0.0003105896175839007, "clip_ratio/low_min": 0.0003105896175839007, "clip_ratio/region_mean": 0.0011767160273545112, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1501.0, "completions/mean_length": 825.683349609375, "completions/mean_terminated_length": 581.219970703125, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.5549363046884537, "epoch": 0.2988338192419825, "frac_reward_zero_std": 0.0, "grad_norm": 0.011654019355773926, "learning_rate": 2e-05, "loss": 0.013636436313390732, "num_tokens": 11047794.0, "reward": 0.03016764484345913, "reward_std": 0.676369845867157, "rewards/correctness/mean": 0.4333333373069763, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.40316569805145264, "rewards/length_penalty/std": 0.2929583787918091, "sampling/importance_sampling_ratio/max": 1.581242561340332, "sampling/importance_sampling_ratio/mean": 0.9807766675949097, "sampling/importance_sampling_ratio/min": 0.5673872232437134, "sampling/sampling_logp_difference/max": 0.5667133331298828, "sampling/sampling_logp_difference/mean": 0.035227056592702866, "step": 205, "step_time": 26.962018608115613 }, { "clip_ratio/high_max": 0.0010171405604827062, "clip_ratio/high_mean": 0.0010171405604827062, "clip_ratio/low_mean": 0.000210162413471456, "clip_ratio/low_min": 0.000210162413471456, "clip_ratio/region_mean": 0.0012273029885060776, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1922.0, "completions/mean_length": 764.550048828125, "completions/mean_terminated_length": 567.09619140625, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 0.44448081652323407, "epoch": 0.30029154518950435, "frac_reward_zero_std": 0.0, "grad_norm": 0.03577873483300209, "learning_rate": 2e-05, "loss": 0.07025798410177231, "num_tokens": 11098027.0, "reward": 0.4600179195404053, "reward_std": 0.6571142673492432, "rewards/correctness/mean": 0.8333333134651184, "rewards/correctness/std": 0.3758230209350586, "rewards/length_penalty/mean": -0.3733154237270355, "rewards/length_penalty/std": 0.3169111907482147, "sampling/importance_sampling_ratio/max": 1.6363152265548706, "sampling/importance_sampling_ratio/mean": 0.9839851260185242, "sampling/importance_sampling_ratio/min": 0.556125283241272, "sampling/sampling_logp_difference/max": 0.5867617130279541, "sampling/sampling_logp_difference/mean": 0.028837379068136215, "step": 206, "step_time": 25.539803754305467 }, { "clip_ratio/high_max": 0.0004262033350338849, "clip_ratio/high_mean": 0.0004262033350338849, "clip_ratio/low_mean": 6.123451991394784e-05, "clip_ratio/low_min": 6.123451991394784e-05, "clip_ratio/region_mean": 0.0004874378549478327, "completions/clipped_ratio": 0.0, "completions/max_length": 1428.0, "completions/max_terminated_length": 1428.0, "completions/mean_length": 527.7333374023438, "completions/mean_terminated_length": 527.7333374023438, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.17581425110499063, "epoch": 0.30174927113702626, "frac_reward_zero_std": 0.0, "grad_norm": 0.010289754718542099, "learning_rate": 2e-05, "loss": 0.036692116409540176, "num_tokens": 11133741.0, "reward": 0.24231772124767303, "reward_std": 0.459685742855072, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.25768229365348816, "rewards/length_penalty/std": 0.1244402751326561, "sampling/importance_sampling_ratio/max": 1.6149274110794067, "sampling/importance_sampling_ratio/mean": 0.9932077527046204, "sampling/importance_sampling_ratio/min": 0.5407243967056274, "sampling/sampling_logp_difference/max": 0.6148456335067749, "sampling/sampling_logp_difference/mean": 0.013211480341851711, "step": 207, "step_time": 17.803234454011545 }, { "clip_ratio/high_max": 0.0016383469458863449, "clip_ratio/high_mean": 0.0016383469458863449, "clip_ratio/low_mean": 0.000521545600349782, "clip_ratio/low_min": 0.000521545600349782, "clip_ratio/region_mean": 0.002159892554724744, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1878.0, "completions/mean_length": 699.7166748046875, "completions/mean_terminated_length": 653.22412109375, "completions/min_length": 160.0, "completions/min_terminated_length": 160.0, "entropy": 0.5753067980209986, "epoch": 0.3032069970845481, "frac_reward_zero_std": 0.0, "grad_norm": 0.02075957879424095, "learning_rate": 2e-05, "loss": -0.012005891650915146, "num_tokens": 11182824.0, "reward": 0.3250081539154053, "reward_std": 0.6316904425621033, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.3416585326194763, "rewards/length_penalty/std": 0.26331615447998047, "sampling/importance_sampling_ratio/max": 1.7022151947021484, "sampling/importance_sampling_ratio/mean": 0.9806972146034241, "sampling/importance_sampling_ratio/min": 0.5493212342262268, "sampling/sampling_logp_difference/max": 0.5990718603134155, "sampling/sampling_logp_difference/mean": 0.036425646394491196, "step": 208, "step_time": 26.843433101894334 }, { "clip_ratio/high_max": 0.002154885674826801, "clip_ratio/high_mean": 0.002154885674826801, "clip_ratio/low_mean": 0.0003000959501756976, "clip_ratio/low_min": 0.0003000959501756976, "clip_ratio/region_mean": 0.0024549816153012216, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1931.0, "completions/mean_length": 752.2833862304688, "completions/mean_terminated_length": 730.322021484375, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 0.47774187723795575, "epoch": 0.30466472303207, "frac_reward_zero_std": 0.0, "grad_norm": 0.021483179181814194, "learning_rate": 2e-05, "loss": 0.04477334022521973, "num_tokens": 11232691.0, "reward": 0.39934083819389343, "reward_std": 0.6035918593406677, "rewards/correctness/mean": 0.7666666507720947, "rewards/correctness/std": 0.42652183771133423, "rewards/length_penalty/mean": -0.3673258423805237, "rewards/length_penalty/std": 0.2375745326280594, "sampling/importance_sampling_ratio/max": 1.7074968814849854, "sampling/importance_sampling_ratio/mean": 0.9838190078735352, "sampling/importance_sampling_ratio/min": 0.5806995034217834, "sampling/sampling_logp_difference/max": 0.5435218811035156, "sampling/sampling_logp_difference/mean": 0.030921122059226036, "step": 209, "step_time": 25.62166529521346 }, { "clip_ratio/high_max": 0.0014697443693876266, "clip_ratio/high_mean": 0.0014697443693876266, "clip_ratio/low_mean": 0.00012251495233310075, "clip_ratio/low_min": 0.00012251495233310075, "clip_ratio/region_mean": 0.0015922593399106215, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1621.0, "completions/mean_length": 818.183349609375, "completions/mean_terminated_length": 797.3389892578125, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.29946333169937134, "epoch": 0.30612244897959184, "frac_reward_zero_std": 0.0, "grad_norm": 0.020209552720189095, "learning_rate": 2e-05, "loss": 0.024181172251701355, "num_tokens": 11287552.0, "reward": 0.25049644708633423, "reward_std": 0.5160675048828125, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.39950358867645264, "rewards/length_penalty/std": 0.17354395985603333, "sampling/importance_sampling_ratio/max": 1.6998443603515625, "sampling/importance_sampling_ratio/mean": 0.9894348382949829, "sampling/importance_sampling_ratio/min": 0.5764414072036743, "sampling/sampling_logp_difference/max": 0.5508816242218018, "sampling/sampling_logp_difference/mean": 0.02073776163160801, "step": 210, "step_time": 26.597167272819206 }, { "clip_ratio/high_max": 0.0006329893512884155, "clip_ratio/high_mean": 0.0006329893512884155, "clip_ratio/low_mean": 0.00010552056725524987, "clip_ratio/low_min": 0.00010552056725524987, "clip_ratio/region_mean": 0.0007385099209689846, "completions/clipped_ratio": 0.0, "completions/max_length": 1228.0, "completions/max_terminated_length": 1228.0, "completions/mean_length": 609.9500122070312, "completions/mean_terminated_length": 609.9500122070312, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "entropy": 0.25365452468395233, "epoch": 0.3075801749271137, "frac_reward_zero_std": 0.0, "grad_norm": 0.013402734883129597, "learning_rate": 2e-05, "loss": 0.0022450629621744156, "num_tokens": 11332689.0, "reward": 0.23550619184970856, "reward_std": 0.5688512921333313, "rewards/correctness/mean": 0.5333333611488342, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.2978271543979645, "rewards/length_penalty/std": 0.15577198565006256, "sampling/importance_sampling_ratio/max": 1.5590828657150269, "sampling/importance_sampling_ratio/mean": 0.9906116127967834, "sampling/importance_sampling_ratio/min": 0.4748865067958832, "sampling/sampling_logp_difference/max": 0.7446794509887695, "sampling/sampling_logp_difference/mean": 0.01836034655570984, "step": 211, "step_time": 16.478536105016246 }, { "clip_ratio/high_max": 0.0009625063539715484, "clip_ratio/high_mean": 0.0009625063539715484, "clip_ratio/low_mean": 0.0002618402722873725, "clip_ratio/low_min": 0.0002618402722873725, "clip_ratio/region_mean": 0.0012243466432361554, "completions/clipped_ratio": 0.31666669249534607, "completions/max_length": 2048.0, "completions/max_terminated_length": 2039.0, "completions/mean_length": 1276.533447265625, "completions/mean_terminated_length": 919.0243530273438, "completions/min_length": 425.0, "completions/min_terminated_length": 425.0, "entropy": 0.4737963179747264, "epoch": 0.30903790087463556, "frac_reward_zero_std": 0.0, "grad_norm": 0.03050401620566845, "learning_rate": 2e-05, "loss": 0.0942547619342804, "num_tokens": 11415301.0, "reward": 0.010026042349636555, "reward_std": 0.7672510147094727, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.6233072876930237, "rewards/length_penalty/std": 0.31783929467201233, "sampling/importance_sampling_ratio/max": 1.8523190021514893, "sampling/importance_sampling_ratio/mean": 0.9841621518135071, "sampling/importance_sampling_ratio/min": 0.55897456407547, "sampling/sampling_logp_difference/max": 0.6164383888244629, "sampling/sampling_logp_difference/mean": 0.030480222776532173, "step": 212, "step_time": 28.276480046100914 }, { "clip_ratio/high_max": 0.0016097433981485665, "clip_ratio/high_mean": 0.0016097433981485665, "clip_ratio/low_mean": 0.0001837208510551136, "clip_ratio/low_min": 0.0001837208510551136, "clip_ratio/region_mean": 0.0017934642576922972, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1869.0, "completions/mean_length": 968.0167236328125, "completions/mean_terminated_length": 930.77587890625, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.373983492453893, "epoch": 0.3104956268221574, "frac_reward_zero_std": 0.0, "grad_norm": 0.02037685178220272, "learning_rate": 2e-05, "loss": 0.021700453013181686, "num_tokens": 11478072.0, "reward": 0.16066895425319672, "reward_std": 0.665320098400116, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.47266438603401184, "rewards/length_penalty/std": 0.2385079562664032, "sampling/importance_sampling_ratio/max": 1.6764709949493408, "sampling/importance_sampling_ratio/mean": 0.9870345592498779, "sampling/importance_sampling_ratio/min": 0.596617579460144, "sampling/sampling_logp_difference/max": 0.5166909694671631, "sampling/sampling_logp_difference/mean": 0.02555643580853939, "step": 213, "step_time": 27.16146021615714 }, { "clip_ratio/high_max": 0.0008683102641953155, "clip_ratio/high_mean": 0.0008683102641953155, "clip_ratio/low_mean": 0.00022832577330215523, "clip_ratio/low_min": 0.00022832577330215523, "clip_ratio/region_mean": 0.0010966360423481092, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1687.0, "completions/mean_length": 755.8167114257812, "completions/mean_terminated_length": 711.2586059570312, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "entropy": 0.39848822603623074, "epoch": 0.3119533527696793, "frac_reward_zero_std": 0.0, "grad_norm": 0.029716582968831062, "learning_rate": 2e-05, "loss": 0.10244863480329514, "num_tokens": 11527351.0, "reward": 0.13094890117645264, "reward_std": 0.6084116697311401, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.36905109882354736, "rewards/length_penalty/std": 0.22060993313789368, "sampling/importance_sampling_ratio/max": 1.7121472358703613, "sampling/importance_sampling_ratio/mean": 0.9863086342811584, "sampling/importance_sampling_ratio/min": 0.5983611345291138, "sampling/sampling_logp_difference/max": 0.5377483367919922, "sampling/sampling_logp_difference/mean": 0.026749493554234505, "step": 214, "step_time": 25.29392878874205 }, { "clip_ratio/high_max": 0.00255237731228893, "clip_ratio/high_mean": 0.00255237731228893, "clip_ratio/low_mean": 0.0001194303622469306, "clip_ratio/low_min": 0.0001194303622469306, "clip_ratio/region_mean": 0.002671807635730753, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1438.0, "completions/mean_length": 673.5000610351562, "completions/mean_terminated_length": 650.203369140625, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.40356265505154926, "epoch": 0.31341107871720114, "frac_reward_zero_std": 0.0, "grad_norm": 0.018645118921995163, "learning_rate": 2e-05, "loss": 0.014171887189149857, "num_tokens": 11572401.0, "reward": 0.5378092527389526, "reward_std": 0.4084661304950714, "rewards/correctness/mean": 0.8666666746139526, "rewards/correctness/std": 0.34280335903167725, "rewards/length_penalty/mean": -0.328857421875, "rewards/length_penalty/std": 0.15821580588817596, "sampling/importance_sampling_ratio/max": 1.9680250883102417, "sampling/importance_sampling_ratio/mean": 0.9855837225914001, "sampling/importance_sampling_ratio/min": 0.5597352981567383, "sampling/sampling_logp_difference/max": 0.6770305633544922, "sampling/sampling_logp_difference/mean": 0.028516538441181183, "step": 215, "step_time": 25.188028177944943 }, { "clip_ratio/high_max": 0.0019273605042447646, "clip_ratio/high_mean": 0.0019273605042447646, "clip_ratio/low_mean": 0.000278434233526544, "clip_ratio/low_min": 0.000278434233526544, "clip_ratio/region_mean": 0.0022057947159434357, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1736.0, "completions/mean_length": 808.7000122070312, "completions/mean_terminated_length": 720.1785888671875, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.5329127709070841, "epoch": 0.31486880466472306, "frac_reward_zero_std": 0.0, "grad_norm": 0.03339240700006485, "learning_rate": 2e-05, "loss": 0.0696830004453659, "num_tokens": 11625453.0, "reward": 0.3551269769668579, "reward_std": 0.5532042980194092, "rewards/correctness/mean": 0.75, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.3948730528354645, "rewards/length_penalty/std": 0.23168165981769562, "sampling/importance_sampling_ratio/max": 1.7921773195266724, "sampling/importance_sampling_ratio/mean": 0.9823166728019714, "sampling/importance_sampling_ratio/min": 0.44885724782943726, "sampling/sampling_logp_difference/max": 0.8010504245758057, "sampling/sampling_logp_difference/mean": 0.03382866829633713, "step": 216, "step_time": 26.147842217236757 }, { "clip_ratio/high_max": 0.0022281459532678127, "clip_ratio/high_mean": 0.0022281459532678127, "clip_ratio/low_mean": 0.0002019470775849186, "clip_ratio/low_min": 0.0002019470775849186, "clip_ratio/region_mean": 0.0024300930284274123, "completions/clipped_ratio": 0.0, "completions/max_length": 1384.0, "completions/max_terminated_length": 1384.0, "completions/mean_length": 588.2333374023438, "completions/mean_terminated_length": 588.2333374023438, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.39598366618156433, "epoch": 0.3163265306122449, "frac_reward_zero_std": 0.0, "grad_norm": 0.016502713784575462, "learning_rate": 2e-05, "loss": -0.003790651448071003, "num_tokens": 11665837.0, "reward": 0.3294433653354645, "reward_std": 0.5549144148826599, "rewards/correctness/mean": 0.6166666746139526, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.28722330927848816, "rewards/length_penalty/std": 0.12038163840770721, "sampling/importance_sampling_ratio/max": 2.0160341262817383, "sampling/importance_sampling_ratio/mean": 0.9862297773361206, "sampling/importance_sampling_ratio/min": 0.5633999705314636, "sampling/sampling_logp_difference/max": 0.7011322975158691, "sampling/sampling_logp_difference/mean": 0.026935497298836708, "step": 217, "step_time": 17.627885991241783 }, { "clip_ratio/high_max": 0.0012065030217248325, "clip_ratio/high_mean": 0.0012065030217248325, "clip_ratio/low_mean": 6.776323910647382e-05, "clip_ratio/low_min": 6.776323910647382e-05, "clip_ratio/region_mean": 0.001274266255980668, "completions/clipped_ratio": 0.0, "completions/max_length": 900.0, "completions/max_terminated_length": 900.0, "completions/mean_length": 495.2333679199219, "completions/mean_terminated_length": 495.2333679199219, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.21478366355101267, "epoch": 0.3177842565597668, "frac_reward_zero_std": 0.0, "grad_norm": 0.014110628515481949, "learning_rate": 2e-05, "loss": 0.02474321611225605, "num_tokens": 11699451.0, "reward": 0.35818687081336975, "reward_std": 0.4749058783054352, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.24181315302848816, "rewards/length_penalty/std": 0.0766502395272255, "sampling/importance_sampling_ratio/max": 1.6567569971084595, "sampling/importance_sampling_ratio/mean": 0.9917597770690918, "sampling/importance_sampling_ratio/min": 0.6078271865844727, "sampling/sampling_logp_difference/max": 0.5048620700836182, "sampling/sampling_logp_difference/mean": 0.01577763631939888, "step": 218, "step_time": 11.986751473043114 }, { "clip_ratio/high_max": 0.0010579859275215615, "clip_ratio/high_mean": 0.0010579859275215615, "clip_ratio/low_mean": 8.969967166194692e-05, "clip_ratio/low_min": 8.969967166194692e-05, "clip_ratio/region_mean": 0.0011476855822062741, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1685.0, "completions/mean_length": 569.75, "completions/mean_terminated_length": 544.6949462890625, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "entropy": 0.33895600338776904, "epoch": 0.31924198250728864, "frac_reward_zero_std": 0.0, "grad_norm": 0.021646374836564064, "learning_rate": 2e-05, "loss": -0.0046263933181762695, "num_tokens": 11738766.0, "reward": 0.3051351010799408, "reward_std": 0.5726703405380249, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.2781982421875, "rewards/length_penalty/std": 0.17928346991539001, "sampling/importance_sampling_ratio/max": 1.765254259109497, "sampling/importance_sampling_ratio/mean": 0.9882793426513672, "sampling/importance_sampling_ratio/min": 0.5402964949607849, "sampling/sampling_logp_difference/max": 0.6156373023986816, "sampling/sampling_logp_difference/mean": 0.024014677852392197, "step": 219, "step_time": 24.96930184075609 }, { "clip_ratio/high_max": 0.0014730089436246392, "clip_ratio/high_mean": 0.0014730089436246392, "clip_ratio/low_mean": 0.0003914697444997728, "clip_ratio/low_min": 0.0003914697444997728, "clip_ratio/region_mean": 0.0018644786614459008, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 2048.0, "completions/max_terminated_length": 1931.0, "completions/mean_length": 1121.033447265625, "completions/mean_terminated_length": 723.7619018554688, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.6407545804977417, "epoch": 0.3206997084548105, "frac_reward_zero_std": 0.0, "grad_norm": 0.03170187398791313, "learning_rate": 2e-05, "loss": 0.06011481583118439, "num_tokens": 11811838.0, "reward": -0.13071289658546448, "reward_std": 0.7018084526062012, "rewards/correctness/mean": 0.4166666567325592, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.5473795533180237, "rewards/length_penalty/std": 0.33772245049476624, "sampling/importance_sampling_ratio/max": 1.9850695133209229, "sampling/importance_sampling_ratio/mean": 0.9797592759132385, "sampling/importance_sampling_ratio/min": 0.5431559681892395, "sampling/sampling_logp_difference/max": 0.6856539249420166, "sampling/sampling_logp_difference/mean": 0.03889491781592369, "step": 220, "step_time": 27.91312867240049 }, { "clip_ratio/high_max": 0.001933912009311219, "clip_ratio/high_mean": 0.001933912009311219, "clip_ratio/low_mean": 0.0001791515969671309, "clip_ratio/low_min": 0.0001791515969671309, "clip_ratio/region_mean": 0.00211306360627835, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1883.0, "completions/mean_length": 804.5833740234375, "completions/mean_terminated_length": 691.54541015625, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.5445303867260615, "epoch": 0.32215743440233235, "frac_reward_zero_std": 0.0, "grad_norm": 0.025696007534861565, "learning_rate": 2e-05, "loss": 0.02477307617664337, "num_tokens": 11866393.0, "reward": 0.4404703974723816, "reward_std": 0.5653824210166931, "rewards/correctness/mean": 0.8333333134651184, "rewards/correctness/std": 0.3758230209350586, "rewards/length_penalty/mean": -0.3928629457950592, "rewards/length_penalty/std": 0.2689432203769684, "sampling/importance_sampling_ratio/max": 2.073718547821045, "sampling/importance_sampling_ratio/mean": 0.9808348417282104, "sampling/importance_sampling_ratio/min": 0.5603417754173279, "sampling/sampling_logp_difference/max": 0.7293434143066406, "sampling/sampling_logp_difference/mean": 0.0363910011947155, "step": 221, "step_time": 26.85885496903211 }, { "clip_ratio/high_max": 0.0008764541513907412, "clip_ratio/high_mean": 0.0008764541513907412, "clip_ratio/low_mean": 0.0004011962834435205, "clip_ratio/low_min": 0.0004011962834435205, "clip_ratio/region_mean": 0.0012776504348342617, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 2014.0, "completions/mean_length": 836.6500244140625, "completions/mean_terminated_length": 594.3800048828125, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.6271700163682302, "epoch": 0.3236151603498542, "frac_reward_zero_std": 0.0, "grad_norm": 0.0441814623773098, "learning_rate": 2e-05, "loss": 0.06094851344823837, "num_tokens": 11922202.0, "reward": 0.3914795219898224, "reward_std": 0.6983773708343506, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.40852051973342896, "rewards/length_penalty/std": 0.3213048577308655, "sampling/importance_sampling_ratio/max": 1.7573819160461426, "sampling/importance_sampling_ratio/mean": 0.9770511388778687, "sampling/importance_sampling_ratio/min": 0.5408254861831665, "sampling/sampling_logp_difference/max": 0.6146585941314697, "sampling/sampling_logp_difference/mean": 0.04339605197310448, "step": 222, "step_time": 26.56876488425769 }, { "clip_ratio/high_max": 0.0017613033123780042, "clip_ratio/high_mean": 0.0017613033123780042, "clip_ratio/low_mean": 0.0005046187434345484, "clip_ratio/low_min": 0.0005046187434345484, "clip_ratio/region_mean": 0.002265922046111276, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1959.0, "completions/mean_length": 816.9833984375, "completions/mean_terminated_length": 705.0726928710938, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.6024239485462507, "epoch": 0.3250728862973761, "frac_reward_zero_std": 0.0, "grad_norm": 0.012940777465701103, "learning_rate": 2e-05, "loss": -0.0055752890184521675, "num_tokens": 11975211.0, "reward": 0.35108238458633423, "reward_std": 0.6246369481086731, "rewards/correctness/mean": 0.75, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.39891764521598816, "rewards/length_penalty/std": 0.24370859563350677, "sampling/importance_sampling_ratio/max": 1.9855324029922485, "sampling/importance_sampling_ratio/mean": 0.9809147119522095, "sampling/importance_sampling_ratio/min": 0.5747635364532471, "sampling/sampling_logp_difference/max": 0.6858870983123779, "sampling/sampling_logp_difference/mean": 0.03687472641468048, "step": 223, "step_time": 25.676563050830737 }, { "clip_ratio/high_max": 0.000711675122147426, "clip_ratio/high_mean": 0.000711675122147426, "clip_ratio/low_mean": 0.00025659507324841496, "clip_ratio/low_min": 0.00025659507324841496, "clip_ratio/region_mean": 0.0009682701978211602, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1723.0, "completions/mean_length": 932.61669921875, "completions/mean_terminated_length": 735.7843627929688, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.4580090343952179, "epoch": 0.32653061224489793, "frac_reward_zero_std": 0.0, "grad_norm": 0.017624840140342712, "learning_rate": 2e-05, "loss": 0.018925407901406288, "num_tokens": 12036188.0, "reward": 0.32795411348342896, "reward_std": 0.6558542251586914, "rewards/correctness/mean": 0.7833333611488342, "rewards/correctness/std": 0.41545021533966064, "rewards/length_penalty/mean": -0.4553792178630829, "rewards/length_penalty/std": 0.27828916907310486, "sampling/importance_sampling_ratio/max": 1.9225527048110962, "sampling/importance_sampling_ratio/mean": 0.9847249388694763, "sampling/importance_sampling_ratio/min": 0.46072283387184143, "sampling/sampling_logp_difference/max": 0.774958610534668, "sampling/sampling_logp_difference/mean": 0.029688727110624313, "step": 224, "step_time": 26.292989216046408 }, { "clip_ratio/high_max": 0.001636358550361668, "clip_ratio/high_mean": 0.001636358550361668, "clip_ratio/low_mean": 0.0004061789610811199, "clip_ratio/low_min": 0.0004061789610811199, "clip_ratio/region_mean": 0.0020425375162934265, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 2034.0, "completions/mean_length": 822.4667358398438, "completions/mean_terminated_length": 711.0545043945312, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.5940494537353516, "epoch": 0.32798833819241985, "frac_reward_zero_std": 0.0, "grad_norm": 0.029587384313344955, "learning_rate": 2e-05, "loss": 0.02185245417058468, "num_tokens": 12089366.0, "reward": 0.04840495064854622, "reward_std": 0.6032300591468811, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.4015950560569763, "rewards/length_penalty/std": 0.28094282746315, "sampling/importance_sampling_ratio/max": 1.9849194288253784, "sampling/importance_sampling_ratio/mean": 0.980197548866272, "sampling/importance_sampling_ratio/min": 0.5235609412193298, "sampling/sampling_logp_difference/max": 0.6855783462524414, "sampling/sampling_logp_difference/mean": 0.039053529500961304, "step": 225, "step_time": 27.039073039777577 }, { "clip_ratio/high_max": 0.0008389840319675083, "clip_ratio/high_mean": 0.0008389840319675083, "clip_ratio/low_mean": 0.00040638809635614354, "clip_ratio/low_min": 0.00040638809635614354, "clip_ratio/region_mean": 0.0012453721040704597, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1998.0, "completions/mean_length": 1029.0833740234375, "completions/mean_terminated_length": 936.4545288085938, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.4797378530104955, "epoch": 0.3294460641399417, "frac_reward_zero_std": 0.0, "grad_norm": 0.018114378675818443, "learning_rate": 2e-05, "loss": 0.036019355058670044, "num_tokens": 12156131.0, "reward": 0.14751790463924408, "reward_std": 0.7106131911277771, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.5024821162223816, "rewards/length_penalty/std": 0.2651490867137909, "sampling/importance_sampling_ratio/max": 2.109445571899414, "sampling/importance_sampling_ratio/mean": 0.984468936920166, "sampling/importance_sampling_ratio/min": 0.5130950212478638, "sampling/sampling_logp_difference/max": 0.7464251518249512, "sampling/sampling_logp_difference/mean": 0.03132457658648491, "step": 226, "step_time": 26.99720329977572 }, { "clip_ratio/high_max": 0.0010982908812972407, "clip_ratio/high_mean": 0.0010982908812972407, "clip_ratio/low_mean": 0.0004632611332150797, "clip_ratio/low_min": 0.0004632611332150797, "clip_ratio/region_mean": 0.0015615520436161507, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 2036.0, "completions/mean_length": 869.7167358398438, "completions/mean_terminated_length": 762.5999755859375, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.4044430951277415, "epoch": 0.33090379008746357, "frac_reward_zero_std": 0.0, "grad_norm": 0.03916185349225998, "learning_rate": 2e-05, "loss": 0.0859319418668747, "num_tokens": 12214774.0, "reward": 0.04200032725930214, "reward_std": 0.6361604332923889, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.4246663451194763, "rewards/length_penalty/std": 0.2859644591808319, "sampling/importance_sampling_ratio/max": 1.9525314569473267, "sampling/importance_sampling_ratio/mean": 0.9864923357963562, "sampling/importance_sampling_ratio/min": 0.45855051279067993, "sampling/sampling_logp_difference/max": 0.7796849012374878, "sampling/sampling_logp_difference/mean": 0.0267694853246212, "step": 227, "step_time": 26.918379310285673 }, { "clip_ratio/high_max": 0.001957010624513108, "clip_ratio/high_mean": 0.001957010624513108, "clip_ratio/low_mean": 0.0005065244040451944, "clip_ratio/low_min": 0.0005065244040451944, "clip_ratio/region_mean": 0.0024635350758520267, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1909.0, "completions/mean_length": 832.0167236328125, "completions/mean_terminated_length": 671.4151000976562, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 0.5889389564593633, "epoch": 0.3323615160349854, "frac_reward_zero_std": 0.0, "grad_norm": 0.029167935252189636, "learning_rate": 2e-05, "loss": 0.030386347323656082, "num_tokens": 12269365.0, "reward": 0.0437418632209301, "reward_std": 0.6567322611808777, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.40625813603401184, "rewards/length_penalty/std": 0.29009753465652466, "sampling/importance_sampling_ratio/max": 1.9824942350387573, "sampling/importance_sampling_ratio/mean": 0.9821353554725647, "sampling/importance_sampling_ratio/min": 0.5115939974784851, "sampling/sampling_logp_difference/max": 0.6843557357788086, "sampling/sampling_logp_difference/mean": 0.035556863993406296, "step": 228, "step_time": 26.43375566485338 }, { "clip_ratio/high_max": 0.0010997809198064108, "clip_ratio/high_mean": 0.0010997809198064108, "clip_ratio/low_mean": 0.0004652429561247118, "clip_ratio/low_min": 0.0004652429561247118, "clip_ratio/region_mean": 0.0015650238880577187, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 2028.0, "completions/mean_length": 1205.8834228515625, "completions/mean_terminated_length": 899.6591186523438, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.7988403389851252, "epoch": 0.3338192419825073, "frac_reward_zero_std": 0.0, "grad_norm": 0.05212988704442978, "learning_rate": 2e-05, "loss": 0.09869799017906189, "num_tokens": 12347568.0, "reward": -0.02214355580508709, "reward_std": 0.7371479272842407, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.5888102054595947, "rewards/length_penalty/std": 0.30402451753616333, "sampling/importance_sampling_ratio/max": 2.004518985748291, "sampling/importance_sampling_ratio/mean": 0.9750184416770935, "sampling/importance_sampling_ratio/min": 0.4957355558872223, "sampling/sampling_logp_difference/max": 0.7017126083374023, "sampling/sampling_logp_difference/mean": 0.04776180163025856, "step": 229, "step_time": 27.904834392946213 }, { "clip_ratio/high_max": 0.001999637965733806, "clip_ratio/high_mean": 0.001999637965733806, "clip_ratio/low_mean": 0.0005678759189322591, "clip_ratio/low_min": 0.0005678759189322591, "clip_ratio/region_mean": 0.002567513865263512, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1904.0, "completions/mean_length": 1012.0833740234375, "completions/mean_terminated_length": 852.7115478515625, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.850202888250351, "epoch": 0.33527696793002915, "frac_reward_zero_std": 0.0, "grad_norm": 0.029848529025912285, "learning_rate": 2e-05, "loss": 0.031832896173000336, "num_tokens": 12413523.0, "reward": 0.30581870675086975, "reward_std": 0.6168164014816284, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.4941813051700592, "rewards/length_penalty/std": 0.26914945244789124, "sampling/importance_sampling_ratio/max": 2.2152163982391357, "sampling/importance_sampling_ratio/mean": 0.9766492247581482, "sampling/importance_sampling_ratio/min": 0.4991942048072815, "sampling/sampling_logp_difference/max": 0.7953500747680664, "sampling/sampling_logp_difference/mean": 0.046491291373968124, "step": 230, "step_time": 27.46939801494591 }, { "clip_ratio/high_max": 0.002505789753437663, "clip_ratio/high_mean": 0.002505789753437663, "clip_ratio/low_mean": 0.0009047959465533495, "clip_ratio/low_min": 0.0009047959465533495, "clip_ratio/region_mean": 0.003410585729094843, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 1982.0, "completions/mean_length": 1303.1500244140625, "completions/mean_terminated_length": 1032.2955322265625, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 1.137136807044347, "epoch": 0.336734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.08267661929130554, "learning_rate": 2e-05, "loss": 0.11150214076042175, "num_tokens": 12497392.0, "reward": 0.03036295808851719, "reward_std": 0.7289160490036011, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.636303722858429, "rewards/length_penalty/std": 0.29076460003852844, "sampling/importance_sampling_ratio/max": 2.102344512939453, "sampling/importance_sampling_ratio/mean": 0.9701412320137024, "sampling/importance_sampling_ratio/min": 0.35432520508766174, "sampling/sampling_logp_difference/max": 1.0375401973724365, "sampling/sampling_logp_difference/mean": 0.059277601540088654, "step": 231, "step_time": 29.063942020758986 }, { "clip_ratio/high_max": 0.0007115313516502889, "clip_ratio/high_mean": 0.0007115313516502889, "clip_ratio/low_mean": 0.0010855030656481783, "clip_ratio/low_min": 0.0010855030656481783, "clip_ratio/region_mean": 0.0017970344245744248, "completions/clipped_ratio": 0.5166667103767395, "completions/max_length": 2048.0, "completions/max_terminated_length": 1827.0, "completions/mean_length": 1472.5833740234375, "completions/mean_terminated_length": 857.4827270507812, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 1.4071051875750225, "epoch": 0.33819241982507287, "frac_reward_zero_std": 0.0, "grad_norm": 0.022981831803917885, "learning_rate": 2e-05, "loss": 0.016554251313209534, "num_tokens": 12590557.0, "reward": -0.4857015311717987, "reward_std": 0.6490283608436584, "rewards/correctness/mean": 0.23333333432674408, "rewards/correctness/std": 0.42652183771133423, "rewards/length_penalty/mean": -0.7190348505973816, "rewards/length_penalty/std": 0.3195290267467499, "sampling/importance_sampling_ratio/max": 2.0253713130950928, "sampling/importance_sampling_ratio/mean": 0.9651959538459778, "sampling/importance_sampling_ratio/min": 0.4816809892654419, "sampling/sampling_logp_difference/max": 0.7304732799530029, "sampling/sampling_logp_difference/mean": 0.06739413738250732, "step": 232, "step_time": 29.819716746686026 }, { "clip_ratio/high_max": 0.001318247979118799, "clip_ratio/high_mean": 0.001318247979118799, "clip_ratio/low_mean": 0.001080508375404558, "clip_ratio/low_min": 0.001080508375404558, "clip_ratio/region_mean": 0.0023987563715005913, "completions/clipped_ratio": 0.40000003576278687, "completions/max_length": 2048.0, "completions/max_terminated_length": 2032.0, "completions/mean_length": 1318.3834228515625, "completions/mean_terminated_length": 831.9722290039062, "completions/min_length": 424.0, "completions/min_terminated_length": 424.0, "entropy": 1.343762109676997, "epoch": 0.3396501457725947, "frac_reward_zero_std": 0.0, "grad_norm": 0.06460174173116684, "learning_rate": 2e-05, "loss": 0.13693822920322418, "num_tokens": 12675950.0, "reward": -0.14374187588691711, "reward_std": 0.7779264450073242, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.6437418460845947, "rewards/length_penalty/std": 0.33115458488464355, "sampling/importance_sampling_ratio/max": 2.4850027561187744, "sampling/importance_sampling_ratio/mean": 0.9656578898429871, "sampling/importance_sampling_ratio/min": 0.35309550166130066, "sampling/sampling_logp_difference/max": 1.041016697883606, "sampling/sampling_logp_difference/mean": 0.06700213998556137, "step": 233, "step_time": 28.557085115928203 }, { "clip_ratio/high_max": 0.00299358779254059, "clip_ratio/high_mean": 0.00299358779254059, "clip_ratio/low_mean": 0.0010919145764394973, "clip_ratio/low_min": 0.0010919145764394973, "clip_ratio/region_mean": 0.0040855023156230645, "completions/clipped_ratio": 0.4166666865348816, "completions/max_length": 2048.0, "completions/max_terminated_length": 1843.0, "completions/mean_length": 1342.86669921875, "completions/mean_terminated_length": 839.2000122070312, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "entropy": 1.6584531863530476, "epoch": 0.34110787172011664, "frac_reward_zero_std": 0.0, "grad_norm": 0.06740152090787888, "learning_rate": 2e-05, "loss": 0.088716521859169, "num_tokens": 12762062.0, "reward": -0.37236329913139343, "reward_std": 0.7628136873245239, "rewards/correctness/mean": 0.28333333134651184, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.6556966304779053, "rewards/length_penalty/std": 0.3523264527320862, "sampling/importance_sampling_ratio/max": 2.0876710414886475, "sampling/importance_sampling_ratio/mean": 0.9621226787567139, "sampling/importance_sampling_ratio/min": 0.4807973802089691, "sampling/sampling_logp_difference/max": 0.7360490560531616, "sampling/sampling_logp_difference/mean": 0.07380722463130951, "step": 234, "step_time": 29.091785049298778 }, { "clip_ratio/high_max": 0.0011721329453090827, "clip_ratio/high_mean": 0.0011721329453090827, "clip_ratio/low_mean": 0.0011346138780936599, "clip_ratio/low_min": 0.0011346138780936599, "clip_ratio/region_mean": 0.002306746842805296, "completions/clipped_ratio": 0.4166666865348816, "completions/max_length": 2048.0, "completions/max_terminated_length": 1982.0, "completions/mean_length": 1386.4500732421875, "completions/mean_terminated_length": 913.914306640625, "completions/min_length": 488.0, "completions/min_terminated_length": 488.0, "entropy": 1.5859358310699463, "epoch": 0.3425655976676385, "frac_reward_zero_std": 0.0, "grad_norm": 0.07174558192491531, "learning_rate": 2e-05, "loss": 0.12129461765289307, "num_tokens": 12850629.0, "reward": -0.1103108748793602, "reward_std": 0.7828435897827148, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.6769775152206421, "rewards/length_penalty/std": 0.3049227297306061, "sampling/importance_sampling_ratio/max": 2.12099027633667, "sampling/importance_sampling_ratio/mean": 0.9638067483901978, "sampling/importance_sampling_ratio/min": 0.4861777722835541, "sampling/sampling_logp_difference/max": 0.7518830299377441, "sampling/sampling_logp_difference/mean": 0.07101098448038101, "step": 235, "step_time": 28.98842448950745 }, { "clip_ratio/high_max": 0.00159054003112639, "clip_ratio/high_mean": 0.00159054003112639, "clip_ratio/low_mean": 0.0007528462544238815, "clip_ratio/low_min": 0.0007528462544238815, "clip_ratio/region_mean": 0.0023433862952515483, "completions/clipped_ratio": 0.18333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 1578.0, "completions/mean_length": 943.7667236328125, "completions/mean_terminated_length": 695.8775634765625, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 1.3598683575789134, "epoch": 0.34402332361516036, "frac_reward_zero_std": 0.0, "grad_norm": 0.10965503752231598, "learning_rate": 2e-05, "loss": 0.17622920870780945, "num_tokens": 12911415.0, "reward": 0.3058431148529053, "reward_std": 0.69382643699646, "rewards/correctness/mean": 0.7666666507720947, "rewards/correctness/std": 0.42652183771133423, "rewards/length_penalty/mean": -0.46082356572151184, "rewards/length_penalty/std": 0.29444029927253723, "sampling/importance_sampling_ratio/max": 2.1121530532836914, "sampling/importance_sampling_ratio/mean": 0.9703739285469055, "sampling/importance_sampling_ratio/min": 0.4570579528808594, "sampling/sampling_logp_difference/max": 0.7829451560974121, "sampling/sampling_logp_difference/mean": 0.05957689881324768, "step": 236, "step_time": 26.24423805787228 }, { "clip_ratio/high_max": 0.001180762832518667, "clip_ratio/high_mean": 0.001180762832518667, "clip_ratio/low_mean": 0.0010836901783477515, "clip_ratio/low_min": 0.0010836901783477515, "clip_ratio/region_mean": 0.002264453020567695, "completions/clipped_ratio": 0.28333336114883423, "completions/max_length": 2048.0, "completions/max_terminated_length": 2038.0, "completions/mean_length": 1035.0667724609375, "completions/mean_terminated_length": 634.6046752929688, "completions/min_length": 102.0, "completions/min_terminated_length": 102.0, "entropy": 1.7620711425940196, "epoch": 0.3454810495626822, "frac_reward_zero_std": 0.0, "grad_norm": 0.035955771803855896, "learning_rate": 2e-05, "loss": 0.09980551898479462, "num_tokens": 12976619.0, "reward": -0.02207031287252903, "reward_std": 0.8842861652374268, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.5054036378860474, "rewards/length_penalty/std": 0.4012928605079651, "sampling/importance_sampling_ratio/max": 2.0781092643737793, "sampling/importance_sampling_ratio/mean": 0.9625434279441833, "sampling/importance_sampling_ratio/min": 0.4947440028190613, "sampling/sampling_logp_difference/max": 0.7314584255218506, "sampling/sampling_logp_difference/mean": 0.07408052682876587, "step": 237, "step_time": 27.121138498187065 }, { "clip_ratio/high_max": 0.0012231385917402804, "clip_ratio/high_mean": 0.0012231385917402804, "clip_ratio/low_mean": 0.0012731718306895345, "clip_ratio/low_min": 0.0012731718306895345, "clip_ratio/region_mean": 0.0024963104127285383, "completions/clipped_ratio": 0.3333333432674408, "completions/max_length": 2048.0, "completions/max_terminated_length": 1978.0, "completions/mean_length": 1384.933349609375, "completions/mean_terminated_length": 1053.4000244140625, "completions/min_length": 82.0, "completions/min_terminated_length": 82.0, "entropy": 2.305878758430481, "epoch": 0.3469387755102041, "frac_reward_zero_std": 0.0, "grad_norm": 0.05286404490470886, "learning_rate": 2e-05, "loss": 0.03028337098658085, "num_tokens": 13066025.0, "reward": -0.4929036796092987, "reward_std": 0.691119909286499, "rewards/correctness/mean": 0.18333333730697632, "rewards/correctness/std": 0.39020493626594543, "rewards/length_penalty/mean": -0.6762369871139526, "rewards/length_penalty/std": 0.343757301568985, "sampling/importance_sampling_ratio/max": 2.154207944869995, "sampling/importance_sampling_ratio/mean": 0.9563285708427429, "sampling/importance_sampling_ratio/min": 0.3947308659553528, "sampling/sampling_logp_difference/max": 0.9295511245727539, "sampling/sampling_logp_difference/mean": 0.08548901230096817, "step": 238, "step_time": 29.21934208786115 }, { "clip_ratio/high_max": 0.0013924787926953286, "clip_ratio/high_mean": 0.0013924787926953286, "clip_ratio/low_mean": 0.0010311339477387567, "clip_ratio/low_min": 0.0010311339477387567, "clip_ratio/region_mean": 0.0024236127307328084, "completions/clipped_ratio": 0.4166666865348816, "completions/max_length": 2048.0, "completions/max_terminated_length": 1910.0, "completions/mean_length": 1572.9500732421875, "completions/mean_terminated_length": 1233.6285400390625, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "entropy": 3.4712435801823935, "epoch": 0.34839650145772594, "frac_reward_zero_std": 0.0, "grad_norm": 0.08978422731161118, "learning_rate": 2e-05, "loss": 0.07387549430131912, "num_tokens": 13166242.0, "reward": -0.5513753294944763, "reward_std": 0.6481100916862488, "rewards/correctness/mean": 0.21666666865348816, "rewards/correctness/std": 0.41545018553733826, "rewards/length_penalty/mean": -0.7680419683456421, "rewards/length_penalty/std": 0.2795812785625458, "sampling/importance_sampling_ratio/max": 2.4968860149383545, "sampling/importance_sampling_ratio/mean": 0.9490771889686584, "sampling/importance_sampling_ratio/min": 0.4284084737300873, "sampling/sampling_logp_difference/max": 0.9150443077087402, "sampling/sampling_logp_difference/mean": 0.09845836460590363, "step": 239, "step_time": 30.380452723708004 }, { "clip_ratio/high_max": 0.001231961156008765, "clip_ratio/high_mean": 0.001231961156008765, "clip_ratio/low_mean": 0.0005487245459031934, "clip_ratio/low_min": 0.0005487245459031934, "clip_ratio/region_mean": 0.001780685755268981, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 2018.0, "completions/mean_length": 1405.683349609375, "completions/mean_terminated_length": 1306.865478515625, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 2.5578393936157227, "epoch": 0.3498542274052478, "frac_reward_zero_std": 0.0, "grad_norm": 0.05633446201682091, "learning_rate": 2e-05, "loss": 0.0446237251162529, "num_tokens": 13256063.0, "reward": -0.4197021722793579, "reward_std": 0.6761198043823242, "rewards/correctness/mean": 0.2666666805744171, "rewards/correctness/std": 0.4459485113620758, "rewards/length_penalty/mean": -0.6863688230514526, "rewards/length_penalty/std": 0.2887873947620392, "sampling/importance_sampling_ratio/max": 2.3874363899230957, "sampling/importance_sampling_ratio/mean": 0.9629252552986145, "sampling/importance_sampling_ratio/min": 0.46249035000801086, "sampling/sampling_logp_difference/max": 0.8702201843261719, "sampling/sampling_logp_difference/mean": 0.07198359817266464, "step": 240, "step_time": 28.984592099208385 }, { "clip_ratio/high_max": 0.0016788721065192174, "clip_ratio/high_mean": 0.0016788721065192174, "clip_ratio/low_mean": 0.0007056732574710622, "clip_ratio/low_min": 0.0007056732574710622, "clip_ratio/region_mean": 0.0023845453785421946, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1845.0, "completions/mean_length": 1011.4500732421875, "completions/mean_terminated_length": 937.4107666015625, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 2.4122679630915322, "epoch": 0.35131195335276966, "frac_reward_zero_std": 0.0, "grad_norm": 0.03903587907552719, "learning_rate": 2e-05, "loss": 0.016027454286813736, "num_tokens": 13320820.0, "reward": -0.11053874343633652, "reward_std": 0.7124947309494019, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014302253723, "rewards/length_penalty/mean": -0.4938720762729645, "rewards/length_penalty/std": 0.2791534960269928, "sampling/importance_sampling_ratio/max": 2.0177085399627686, "sampling/importance_sampling_ratio/mean": 0.9618121385574341, "sampling/importance_sampling_ratio/min": 0.4745693504810333, "sampling/sampling_logp_difference/max": 0.7453474998474121, "sampling/sampling_logp_difference/mean": 0.07442660629749298, "step": 241, "step_time": 27.092493267264217 }, { "clip_ratio/high_max": 0.002312283574913939, "clip_ratio/high_mean": 0.002312283574913939, "clip_ratio/low_mean": 0.0005130037219108393, "clip_ratio/low_min": 0.0005130037219108393, "clip_ratio/region_mean": 0.0028252873259286084, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1798.0, "completions/mean_length": 1273.7000732421875, "completions/mean_terminated_length": 1137.058837890625, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 3.2736703554789224, "epoch": 0.35276967930029157, "frac_reward_zero_std": 0.0, "grad_norm": 0.05247347056865692, "learning_rate": 2e-05, "loss": 0.06009586527943611, "num_tokens": 13406492.0, "reward": -0.42192384600639343, "reward_std": 0.5470790266990662, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.6219238042831421, "rewards/length_penalty/std": 0.2221079170703888, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9567967057228088, "sampling/importance_sampling_ratio/min": 0.42208459973335266, "sampling/sampling_logp_difference/max": 1.109330415725708, "sampling/sampling_logp_difference/mean": 0.08482299745082855, "step": 242, "step_time": 30.896962225437164 }, { "clip_ratio/high_max": 0.0013816150409790378, "clip_ratio/high_mean": 0.0013816150409790378, "clip_ratio/low_mean": 0.0006224951745631794, "clip_ratio/low_min": 0.0006224951745631794, "clip_ratio/region_mean": 0.0020041101767371097, "completions/clipped_ratio": 0.18333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 2032.0, "completions/mean_length": 1273.783447265625, "completions/mean_terminated_length": 1099.9796142578125, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "entropy": 3.2201629479726157, "epoch": 0.35422740524781343, "frac_reward_zero_std": 0.0, "grad_norm": 0.04824603721499443, "learning_rate": 2e-05, "loss": 0.00270158052444458, "num_tokens": 13487209.0, "reward": -0.5886312127113342, "reward_std": 0.36058759689331055, "rewards/correctness/mean": 0.03333333507180214, "rewards/correctness/std": 0.1810203194618225, "rewards/length_penalty/mean": -0.6219645142555237, "rewards/length_penalty/std": 0.3073245584964752, "sampling/importance_sampling_ratio/max": 1.9463567733764648, "sampling/importance_sampling_ratio/mean": 0.9611196517944336, "sampling/importance_sampling_ratio/min": 0.4671388566493988, "sampling/sampling_logp_difference/max": 0.7611287832260132, "sampling/sampling_logp_difference/mean": 0.07519359886646271, "step": 243, "step_time": 28.869601068086922 }, { "clip_ratio/high_max": 0.0015847850445425138, "clip_ratio/high_mean": 0.0015847850445425138, "clip_ratio/low_mean": 0.0005943572396063246, "clip_ratio/low_min": 0.0005943572396063246, "clip_ratio/region_mean": 0.0021791422429184117, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1803.0, "completions/mean_length": 954.9667358398438, "completions/mean_terminated_length": 786.8077392578125, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 3.0321929454803467, "epoch": 0.3556851311953353, "frac_reward_zero_std": 0.0, "grad_norm": 0.05698322877287865, "learning_rate": 2e-05, "loss": 0.06744833290576935, "num_tokens": 13548177.0, "reward": -0.032958984375, "reward_std": 0.6791627407073975, "rewards/correctness/mean": 0.4333333373069763, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.4662923216819763, "rewards/length_penalty/std": 0.2719915211200714, "sampling/importance_sampling_ratio/max": 2.3848884105682373, "sampling/importance_sampling_ratio/mean": 0.9633015990257263, "sampling/importance_sampling_ratio/min": 0.2515409588813782, "sampling/sampling_logp_difference/max": 1.380149483680725, "sampling/sampling_logp_difference/mean": 0.07275110483169556, "step": 244, "step_time": 26.249029335798696 }, { "clip_ratio/high_max": 0.001650075544603169, "clip_ratio/high_mean": 0.001650075544603169, "clip_ratio/low_mean": 0.0005634138506138697, "clip_ratio/low_min": 0.0005634138506138697, "clip_ratio/region_mean": 0.0022134893806651235, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1696.0, "completions/mean_length": 934.6666870117188, "completions/mean_terminated_length": 876.0701904296875, "completions/min_length": 417.0, "completions/min_terminated_length": 417.0, "entropy": 3.377176562945048, "epoch": 0.35714285714285715, "frac_reward_zero_std": 0.0, "grad_norm": 0.04669371619820595, "learning_rate": 2e-05, "loss": 0.0343376025557518, "num_tokens": 13609727.0, "reward": -0.3063802123069763, "reward_std": 0.4460967779159546, "rewards/correctness/mean": 0.15000000596046448, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.4563802182674408, "rewards/length_penalty/std": 0.1842193901538849, "sampling/importance_sampling_ratio/max": 2.21295166015625, "sampling/importance_sampling_ratio/mean": 0.9604106545448303, "sampling/importance_sampling_ratio/min": 0.40263572335243225, "sampling/sampling_logp_difference/max": 0.9097230434417725, "sampling/sampling_logp_difference/mean": 0.0766955316066742, "step": 245, "step_time": 26.70455032819882 }, { "clip_ratio/high_max": 0.001349970019267251, "clip_ratio/high_mean": 0.001349970019267251, "clip_ratio/low_mean": 0.0006520937652870392, "clip_ratio/low_min": 0.0006520937652870392, "clip_ratio/region_mean": 0.002002063770002375, "completions/clipped_ratio": 0.31666669249534607, "completions/max_length": 2048.0, "completions/max_terminated_length": 1597.0, "completions/mean_length": 1331.6334228515625, "completions/mean_terminated_length": 999.6585083007812, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 3.880954702695211, "epoch": 0.358600583090379, "frac_reward_zero_std": 0.0, "grad_norm": 0.06623799353837967, "learning_rate": 2e-05, "loss": 0.06245960295200348, "num_tokens": 13695215.0, "reward": -0.4002116024494171, "reward_std": 0.5813007354736328, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.6502115726470947, "rewards/length_penalty/std": 0.27233096957206726, "sampling/importance_sampling_ratio/max": 2.084963798522949, "sampling/importance_sampling_ratio/mean": 0.960466206073761, "sampling/importance_sampling_ratio/min": 0.36336299777030945, "sampling/sampling_logp_difference/max": 1.0123529434204102, "sampling/sampling_logp_difference/mean": 0.07546965777873993, "step": 246, "step_time": 29.216931626433507 }, { "clip_ratio/high_max": 0.0011847001054168989, "clip_ratio/high_mean": 0.0011847001054168989, "clip_ratio/low_mean": 0.0007777670446860915, "clip_ratio/low_min": 0.0007777670446860915, "clip_ratio/region_mean": 0.0019624671743561826, "completions/clipped_ratio": 0.28333336114883423, "completions/max_length": 2048.0, "completions/max_terminated_length": 1774.0, "completions/mean_length": 1200.2166748046875, "completions/mean_terminated_length": 865.0465087890625, "completions/min_length": 401.0, "completions/min_terminated_length": 401.0, "entropy": 4.48484206199646, "epoch": 0.36005830903790087, "frac_reward_zero_std": 0.0, "grad_norm": 0.0447349026799202, "learning_rate": 2e-05, "loss": 0.0346875861287117, "num_tokens": 13771848.0, "reward": -0.20270997285842896, "reward_std": 0.7135390043258667, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014302253723, "rewards/length_penalty/mean": -0.5860432982444763, "rewards/length_penalty/std": 0.30049389600753784, "sampling/importance_sampling_ratio/max": 2.260479211807251, "sampling/importance_sampling_ratio/mean": 0.9561495184898376, "sampling/importance_sampling_ratio/min": 0.46121808886528015, "sampling/sampling_logp_difference/max": 0.8155767917633057, "sampling/sampling_logp_difference/mean": 0.08189760893583298, "step": 247, "step_time": 27.50129246409051 }, { "clip_ratio/high_max": 0.0013290448987390846, "clip_ratio/high_mean": 0.0013290448987390846, "clip_ratio/low_mean": 0.0007051646389300004, "clip_ratio/low_min": 0.0007051646389300004, "clip_ratio/region_mean": 0.002034209513415893, "completions/clipped_ratio": 0.38333335518836975, "completions/max_length": 2048.0, "completions/max_terminated_length": 1820.0, "completions/mean_length": 1344.033447265625, "completions/mean_terminated_length": 906.4324340820312, "completions/min_length": 476.0, "completions/min_terminated_length": 476.0, "entropy": 3.829337000846863, "epoch": 0.36151603498542273, "frac_reward_zero_std": 0.0, "grad_norm": 0.04449871927499771, "learning_rate": 2e-05, "loss": 0.10753854364156723, "num_tokens": 13856770.0, "reward": -0.3562662899494171, "reward_std": 0.6584044098854065, "rewards/correctness/mean": 0.30000001192092896, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.6562662720680237, "rewards/length_penalty/std": 0.3033653199672699, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9617136120796204, "sampling/importance_sampling_ratio/min": 0.34381428360939026, "sampling/sampling_logp_difference/max": 1.4782772064208984, "sampling/sampling_logp_difference/mean": 0.07543467730283737, "step": 248, "step_time": 28.84767078841105 }, { "clip_ratio/high_max": 0.0012291103145495679, "clip_ratio/high_mean": 0.0012291103145495679, "clip_ratio/low_mean": 0.0005197829644506177, "clip_ratio/low_min": 0.0005197829644506177, "clip_ratio/region_mean": 0.0017488933032533775, "completions/clipped_ratio": 0.3333333432674408, "completions/max_length": 2048.0, "completions/max_terminated_length": 1760.0, "completions/mean_length": 1211.2333984375, "completions/mean_terminated_length": 792.8500366210938, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 3.8396174112955728, "epoch": 0.3629737609329446, "frac_reward_zero_std": 0.0, "grad_norm": 0.05135185644030571, "learning_rate": 2e-05, "loss": 0.09814514219760895, "num_tokens": 13934414.0, "reward": -0.2747558653354645, "reward_std": 0.6956694722175598, "rewards/correctness/mean": 0.3166666626930237, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.5914224982261658, "rewards/length_penalty/std": 0.3177606463432312, "sampling/importance_sampling_ratio/max": 2.2708933353424072, "sampling/importance_sampling_ratio/mean": 0.9641146659851074, "sampling/importance_sampling_ratio/min": 0.39657682180404663, "sampling/sampling_logp_difference/max": 0.9248855710029602, "sampling/sampling_logp_difference/mean": 0.0686083734035492, "step": 249, "step_time": 27.669461929937825 }, { "clip_ratio/high_max": 0.001416863427342226, "clip_ratio/high_mean": 0.001416863427342226, "clip_ratio/low_mean": 0.0004537880207256724, "clip_ratio/low_min": 0.0004537880207256724, "clip_ratio/region_mean": 0.0018706514577691753, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1674.0, "completions/mean_length": 1387.5167236328125, "completions/mean_terminated_length": 847.1212158203125, "completions/min_length": 464.0, "completions/min_terminated_length": 464.0, "entropy": 3.507745385169983, "epoch": 0.36443148688046645, "frac_reward_zero_std": 0.0, "grad_norm": 0.049529798328876495, "learning_rate": 2e-05, "loss": 0.03842547908425331, "num_tokens": 14023405.0, "reward": -0.5774983763694763, "reward_std": 0.4817473590373993, "rewards/correctness/mean": 0.10000000149011612, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.6774984002113342, "rewards/length_penalty/std": 0.3129677176475525, "sampling/importance_sampling_ratio/max": 1.9605309963226318, "sampling/importance_sampling_ratio/mean": 0.9666308164596558, "sampling/importance_sampling_ratio/min": 0.009267814457416534, "sampling/sampling_logp_difference/max": 4.681207656860352, "sampling/sampling_logp_difference/mean": 0.06255052238702774, "step": 250, "step_time": 29.947691671084613 }, { "clip_ratio/high_max": 0.0006527024427972113, "clip_ratio/high_mean": 0.0006527024427972113, "clip_ratio/low_mean": 0.0007506811040608833, "clip_ratio/low_min": 0.0007506811040608833, "clip_ratio/region_mean": 0.0014033836002151172, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 2014.0, "completions/mean_length": 1425.7000732421875, "completions/mean_terminated_length": 916.5454711914062, "completions/min_length": 440.0, "completions/min_terminated_length": 440.0, "entropy": 3.8734008073806763, "epoch": 0.36588921282798836, "frac_reward_zero_std": 0.0, "grad_norm": 0.05209678038954735, "learning_rate": 2e-05, "loss": 0.16047927737236023, "num_tokens": 14113717.0, "reward": -0.3794759213924408, "reward_std": 0.7310908436775208, "rewards/correctness/mean": 0.3166666626930237, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.6961425542831421, "rewards/length_penalty/std": 0.3152093291282654, "sampling/importance_sampling_ratio/max": 1.9092434644699097, "sampling/importance_sampling_ratio/mean": 0.9646206498146057, "sampling/importance_sampling_ratio/min": 0.28598257899284363, "sampling/sampling_logp_difference/max": 1.2518243789672852, "sampling/sampling_logp_difference/mean": 0.06762811541557312, "step": 251, "step_time": 28.75476169399917 }, { "clip_ratio/high_max": 0.0008625702951879551, "clip_ratio/high_mean": 0.0008625702951879551, "clip_ratio/low_mean": 0.0005921252547220016, "clip_ratio/low_min": 0.0005921252547220016, "clip_ratio/region_mean": 0.0014546955256567646, "completions/clipped_ratio": 0.6166666746139526, "completions/max_length": 2048.0, "completions/max_terminated_length": 1950.0, "completions/mean_length": 1599.6500244140625, "completions/mean_terminated_length": 878.3912963867188, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "entropy": 3.53311018149058, "epoch": 0.3673469387755102, "frac_reward_zero_std": 0.0, "grad_norm": 0.05373596400022507, "learning_rate": 2e-05, "loss": 0.07551965117454529, "num_tokens": 14214006.0, "reward": -0.6310791373252869, "reward_std": 0.5388507843017578, "rewards/correctness/mean": 0.15000000596046448, "rewards/correctness/std": 0.36008477210998535, "rewards/length_penalty/mean": -0.781079113483429, "rewards/length_penalty/std": 0.31191542744636536, "sampling/importance_sampling_ratio/max": 1.9932175874710083, "sampling/importance_sampling_ratio/mean": 0.9672702550888062, "sampling/importance_sampling_ratio/min": 0.00018321472452953458, "sampling/sampling_logp_difference/max": 8.604851722717285, "sampling/sampling_logp_difference/mean": 0.06234521046280861, "step": 252, "step_time": 30.332884459057823 }, { "clip_ratio/high_max": 0.0007697062295240661, "clip_ratio/high_mean": 0.0007697062295240661, "clip_ratio/low_mean": 0.0003675292197537298, "clip_ratio/low_min": 0.0003675292197537298, "clip_ratio/region_mean": 0.0011372354541284342, "completions/clipped_ratio": 0.46666669845581055, "completions/max_length": 2048.0, "completions/max_terminated_length": 1707.0, "completions/mean_length": 1486.11669921875, "completions/mean_terminated_length": 994.46875, "completions/min_length": 457.0, "completions/min_terminated_length": 457.0, "entropy": 3.6248666842778525, "epoch": 0.3688046647230321, "frac_reward_zero_std": 0.0, "grad_norm": 0.04118891432881355, "learning_rate": 2e-05, "loss": 0.04940975084900856, "num_tokens": 14308343.0, "reward": -0.6423096060752869, "reward_std": 0.4567861557006836, "rewards/correctness/mean": 0.0833333358168602, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.7256429195404053, "rewards/length_penalty/std": 0.2870199978351593, "sampling/importance_sampling_ratio/max": 1.9858471155166626, "sampling/importance_sampling_ratio/mean": 0.9671781063079834, "sampling/importance_sampling_ratio/min": 0.00528754061087966, "sampling/sampling_logp_difference/max": 5.242402076721191, "sampling/sampling_logp_difference/mean": 0.06278426945209503, "step": 253, "step_time": 29.397143837297335 }, { "clip_ratio/high_max": 0.000661301325938742, "clip_ratio/high_mean": 0.000661301325938742, "clip_ratio/low_mean": 0.0007718848743631194, "clip_ratio/low_min": 0.0007718848743631194, "clip_ratio/region_mean": 0.0014331861942385633, "completions/clipped_ratio": 0.5666667222976685, "completions/max_length": 2048.0, "completions/max_terminated_length": 1498.0, "completions/mean_length": 1512.0667724609375, "completions/mean_terminated_length": 811.2307739257812, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 3.8703293005625405, "epoch": 0.37026239067055394, "frac_reward_zero_std": 0.0, "grad_norm": 0.052145250141620636, "learning_rate": 2e-05, "loss": 0.051404088735580444, "num_tokens": 14404567.0, "reward": -0.654980480670929, "reward_std": 0.4571812152862549, "rewards/correctness/mean": 0.0833333358168602, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.7383137941360474, "rewards/length_penalty/std": 0.3189006447792053, "sampling/importance_sampling_ratio/max": 2.6937992572784424, "sampling/importance_sampling_ratio/mean": 0.9659257531166077, "sampling/importance_sampling_ratio/min": 0.4248041808605194, "sampling/sampling_logp_difference/max": 0.9909524917602539, "sampling/sampling_logp_difference/mean": 0.0636083111166954, "step": 254, "step_time": 29.787321127718315 }, { "clip_ratio/high_max": 0.0003240131075775328, "clip_ratio/high_mean": 0.0003240131075775328, "clip_ratio/low_mean": 0.0006996982653314868, "clip_ratio/low_min": 0.0006996982653314868, "clip_ratio/region_mean": 0.0010237113941305627, "completions/clipped_ratio": 0.6666666865348816, "completions/max_length": 2048.0, "completions/max_terminated_length": 1046.0, "completions/mean_length": 1509.0833740234375, "completions/mean_terminated_length": 431.25, "completions/min_length": 129.0, "completions/min_terminated_length": 129.0, "entropy": 3.574760834376017, "epoch": 0.3717201166180758, "frac_reward_zero_std": 0.0, "grad_norm": 0.046962328255176544, "learning_rate": 2e-05, "loss": 0.10693094879388809, "num_tokens": 14501352.0, "reward": -0.6701904535293579, "reward_std": 0.497560054063797, "rewards/correctness/mean": 0.06666667014360428, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.7368571162223816, "rewards/length_penalty/std": 0.3835466802120209, "sampling/importance_sampling_ratio/max": 2.203528881072998, "sampling/importance_sampling_ratio/mean": 0.968522846698761, "sampling/importance_sampling_ratio/min": 0.186152845621109, "sampling/sampling_logp_difference/max": 1.6811871528625488, "sampling/sampling_logp_difference/mean": 0.05907248333096504, "step": 255, "step_time": 30.19741909368895 }, { "clip_ratio/high_max": 0.0007360525147911782, "clip_ratio/high_mean": 0.0007360525147911782, "clip_ratio/low_mean": 0.0005556627147598192, "clip_ratio/low_min": 0.0005556627147598192, "clip_ratio/region_mean": 0.0012917152198497206, "completions/clipped_ratio": 0.46666669845581055, "completions/max_length": 2048.0, "completions/max_terminated_length": 1227.0, "completions/mean_length": 1313.683349609375, "completions/mean_terminated_length": 671.15625, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 3.250603199005127, "epoch": 0.37317784256559766, "frac_reward_zero_std": 0.0, "grad_norm": 0.05382511764764786, "learning_rate": 2e-05, "loss": 0.15781483054161072, "num_tokens": 14584073.0, "reward": -0.39144694805145264, "reward_std": 0.6572251915931702, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.6414469480514526, "rewards/length_penalty/std": 0.34851133823394775, "sampling/importance_sampling_ratio/max": 2.388401985168457, "sampling/importance_sampling_ratio/mean": 0.969161868095398, "sampling/importance_sampling_ratio/min": 0.06040038540959358, "sampling/sampling_logp_difference/max": 2.806759834289551, "sampling/sampling_logp_difference/mean": 0.05853750929236412, "step": 256, "step_time": 28.159950858913362 }, { "clip_ratio/high_max": 0.0012178392935311422, "clip_ratio/high_mean": 0.0012178392935311422, "clip_ratio/low_mean": 0.00020864777009895383, "clip_ratio/low_min": 0.00020864777009895383, "clip_ratio/region_mean": 0.0014264870357389252, "completions/clipped_ratio": 0.7500000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 819.0, "completions/mean_length": 1664.5001220703125, "completions/mean_terminated_length": 514.0, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 3.716649810473124, "epoch": 0.3746355685131195, "frac_reward_zero_std": 0.0, "grad_norm": 0.041214779019355774, "learning_rate": 2e-05, "loss": 0.1556004285812378, "num_tokens": 14692163.0, "reward": -0.7127441763877869, "reward_std": 0.5675655603408813, "rewards/correctness/mean": 0.10000000149011612, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.812744140625, "rewards/length_penalty/std": 0.3295469284057617, "sampling/importance_sampling_ratio/max": 2.3230655193328857, "sampling/importance_sampling_ratio/mean": 0.9684692025184631, "sampling/importance_sampling_ratio/min": 0.22633357346057892, "sampling/sampling_logp_difference/max": 1.4857454299926758, "sampling/sampling_logp_difference/mean": 0.05813281610608101, "step": 257, "step_time": 32.608605664689094 }, { "clip_ratio/high_max": 0.0004088454127971393, "clip_ratio/high_mean": 0.0004088454127971393, "clip_ratio/low_mean": 0.000572516568354331, "clip_ratio/low_min": 0.000572516568354331, "clip_ratio/region_mean": 0.0009813619932780664, "completions/clipped_ratio": 0.6833333969116211, "completions/max_length": 2048.0, "completions/max_terminated_length": 1758.0, "completions/mean_length": 1694.466796875, "completions/mean_terminated_length": 931.5789794921875, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 3.7104485034942627, "epoch": 0.3760932944606414, "frac_reward_zero_std": 0.0, "grad_norm": 0.05458291247487068, "learning_rate": 2e-05, "loss": 0.020179079845547676, "num_tokens": 14799421.0, "reward": -0.7773763537406921, "reward_std": 0.3627850115299225, "rewards/correctness/mean": 0.05000000074505806, "rewards/correctness/std": 0.21978417038917542, "rewards/length_penalty/mean": -0.8273763060569763, "rewards/length_penalty/std": 0.2802988290786743, "sampling/importance_sampling_ratio/max": 2.6504411697387695, "sampling/importance_sampling_ratio/mean": 0.9683261513710022, "sampling/importance_sampling_ratio/min": 0.19380559027194977, "sampling/sampling_logp_difference/max": 1.640899658203125, "sampling/sampling_logp_difference/mean": 0.05906633287668228, "step": 258, "step_time": 31.266251852735877 }, { "clip_ratio/high_max": 0.0009903069585561752, "clip_ratio/high_mean": 0.0009903069585561752, "clip_ratio/low_mean": 0.0003805569576798007, "clip_ratio/low_min": 0.0003805569576798007, "clip_ratio/region_mean": 0.0013708639307878911, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 1222.0, "completions/mean_length": 1482.9334716796875, "completions/mean_terminated_length": 635.3333740234375, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 2.8029869397481284, "epoch": 0.37755102040816324, "frac_reward_zero_std": 0.0, "grad_norm": 0.05087699368596077, "learning_rate": 2e-05, "loss": 0.08635292947292328, "num_tokens": 14893447.0, "reward": -0.5407552123069763, "reward_std": 0.6088138222694397, "rewards/correctness/mean": 0.18333333730697632, "rewards/correctness/std": 0.39020493626594543, "rewards/length_penalty/mean": -0.7240885496139526, "rewards/length_penalty/std": 0.34891748428344727, "sampling/importance_sampling_ratio/max": 2.793879270553589, "sampling/importance_sampling_ratio/mean": 0.9743561744689941, "sampling/importance_sampling_ratio/min": 0.42227956652641296, "sampling/sampling_logp_difference/max": 1.0274310111999512, "sampling/sampling_logp_difference/mean": 0.04804966226220131, "step": 259, "step_time": 29.22046184912324 }, { "clip_ratio/high_max": 0.0002077679346257355, "clip_ratio/high_mean": 0.0002077679346257355, "clip_ratio/low_mean": 0.0005443845293484628, "clip_ratio/low_min": 0.0005443845293484628, "clip_ratio/region_mean": 0.0007521524627615387, "completions/clipped_ratio": 0.7000000476837158, "completions/max_length": 2048.0, "completions/max_terminated_length": 778.0, "completions/mean_length": 1533.416748046875, "completions/mean_terminated_length": 332.72222900390625, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 4.107400139172872, "epoch": 0.37900874635568516, "frac_reward_zero_std": 0.0, "grad_norm": 0.04479505494236946, "learning_rate": 2e-05, "loss": 0.015183070674538612, "num_tokens": 14989762.0, "reward": -0.5487386584281921, "reward_std": 0.7084459662437439, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.7487385869026184, "rewards/length_penalty/std": 0.38978472352027893, "sampling/importance_sampling_ratio/max": 2.0737600326538086, "sampling/importance_sampling_ratio/mean": 0.9648609757423401, "sampling/importance_sampling_ratio/min": 0.04329626262187958, "sampling/sampling_logp_difference/max": 3.1396889686584473, "sampling/sampling_logp_difference/mean": 0.0641050934791565, "step": 260, "step_time": 30.14038326102309 }, { "clip_ratio/high_max": 0.0007714536768617108, "clip_ratio/high_mean": 0.0007714536768617108, "clip_ratio/low_mean": 0.00036042986418275785, "clip_ratio/low_min": 0.00036042986418275785, "clip_ratio/region_mean": 0.0011318835507457454, "completions/clipped_ratio": 0.7833333611488342, "completions/max_length": 2048.0, "completions/max_terminated_length": 1502.0, "completions/mean_length": 1756.7501220703125, "completions/mean_terminated_length": 703.769287109375, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "entropy": 3.2599937518437705, "epoch": 0.380466472303207, "frac_reward_zero_std": 0.0, "grad_norm": 0.05582372471690178, "learning_rate": 2e-05, "loss": 0.05368804931640625, "num_tokens": 15100477.0, "reward": -0.8244547843933105, "reward_std": 0.35956335067749023, "rewards/correctness/mean": 0.03333333507180214, "rewards/correctness/std": 0.1810203194618225, "rewards/length_penalty/mean": -0.8577880859375, "rewards/length_penalty/std": 0.2812116742134094, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9717200398445129, "sampling/importance_sampling_ratio/min": 0.19961509108543396, "sampling/sampling_logp_difference/max": 1.6113643646240234, "sampling/sampling_logp_difference/mean": 0.0515742264688015, "step": 261, "step_time": 30.495931137586012 }, { "clip_ratio/high_max": 0.0002528037409016785, "clip_ratio/high_mean": 0.0002528037409016785, "clip_ratio/low_mean": 0.0004106710111955181, "clip_ratio/low_min": 0.0004106710111955181, "clip_ratio/region_mean": 0.0006634747648301224, "completions/clipped_ratio": 0.8333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 1273.0, "completions/mean_length": 1826.783447265625, "completions/mean_terminated_length": 720.7000122070312, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 3.512176593144735, "epoch": 0.3819241982507289, "frac_reward_zero_std": 0.0, "grad_norm": 0.05167768523097038, "learning_rate": 2e-05, "loss": 0.14026087522506714, "num_tokens": 15216964.0, "reward": -0.7586507797241211, "reward_std": 0.5336034893989563, "rewards/correctness/mean": 0.13333334028720856, "rewards/correctness/std": 0.34280335903167725, "rewards/length_penalty/mean": -0.8919840455055237, "rewards/length_penalty/std": 0.24925105273723602, "sampling/importance_sampling_ratio/max": 2.870110034942627, "sampling/importance_sampling_ratio/mean": 0.970743715763092, "sampling/importance_sampling_ratio/min": 0.05179512873291969, "sampling/sampling_logp_difference/max": 2.9604592323303223, "sampling/sampling_logp_difference/mean": 0.05326750501990318, "step": 262, "step_time": 31.836842478020117 }, { "clip_ratio/high_max": 0.0010552301925296585, "clip_ratio/high_mean": 0.0010552301925296585, "clip_ratio/low_mean": 0.0001456152749597095, "clip_ratio/low_min": 0.0001456152749597095, "clip_ratio/region_mean": 0.0012008454650640488, "completions/clipped_ratio": 0.8833333849906921, "completions/max_length": 2048.0, "completions/max_terminated_length": 1374.0, "completions/mean_length": 1880.800048828125, "completions/mean_terminated_length": 614.857177734375, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 3.755980928738912, "epoch": 0.38338192419825073, "frac_reward_zero_std": 0.0, "grad_norm": 0.0540660098195076, "learning_rate": 2e-05, "loss": 0.08667202293872833, "num_tokens": 15333742.0, "reward": -0.8016927242279053, "reward_std": 0.49989140033721924, "rewards/correctness/mean": 0.11666666716337204, "rewards/correctness/std": 0.32373178005218506, "rewards/length_penalty/mean": -0.9183593988418579, "rewards/length_penalty/std": 0.23502227663993835, "sampling/importance_sampling_ratio/max": 2.7149598598480225, "sampling/importance_sampling_ratio/mean": 0.9687901139259338, "sampling/importance_sampling_ratio/min": 0.2677181661128998, "sampling/sampling_logp_difference/max": 1.3178205490112305, "sampling/sampling_logp_difference/mean": 0.05672689154744148, "step": 263, "step_time": 30.868781251832843 }, { "clip_ratio/high_max": 0.00029544224283502746, "clip_ratio/high_mean": 0.00029544224283502746, "clip_ratio/low_mean": 0.0004125194827793166, "clip_ratio/low_min": 0.0004125194827793166, "clip_ratio/region_mean": 0.0007079617207637057, "completions/clipped_ratio": 0.7000000476837158, "completions/max_length": 2048.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 1585.166748046875, "completions/mean_terminated_length": 505.22222900390625, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 4.44981849193573, "epoch": 0.3848396501457726, "frac_reward_zero_std": 0.0, "grad_norm": 0.059756286442279816, "learning_rate": 2e-05, "loss": 0.2277233600616455, "num_tokens": 15433112.0, "reward": -0.5573405027389526, "reward_std": 0.7350220084190369, "rewards/correctness/mean": 0.21666666865348816, "rewards/correctness/std": 0.41545018553733826, "rewards/length_penalty/mean": -0.7740071415901184, "rewards/length_penalty/std": 0.3540513217449188, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9641476273536682, "sampling/importance_sampling_ratio/min": 0.21773481369018555, "sampling/sampling_logp_difference/max": 1.524477481842041, "sampling/sampling_logp_difference/mean": 0.06671612709760666, "step": 264, "step_time": 30.480364996707067 }, { "clip_ratio/high_max": 0.0003839943140822773, "clip_ratio/high_mean": 0.0003839943140822773, "clip_ratio/low_mean": 0.0004310507780852883, "clip_ratio/low_min": 0.0004310507780852883, "clip_ratio/region_mean": 0.0008150450836789483, "completions/clipped_ratio": 0.7166666984558105, "completions/max_length": 2048.0, "completions/max_terminated_length": 1209.0, "completions/mean_length": 1629.800048828125, "completions/mean_terminated_length": 572.0, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 3.6476961374282837, "epoch": 0.38629737609329445, "frac_reward_zero_std": 0.0, "grad_norm": 0.06591147929430008, "learning_rate": 2e-05, "loss": 0.17030805349349976, "num_tokens": 15533870.0, "reward": -0.5958008170127869, "reward_std": 0.7002316117286682, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.7958008050918579, "rewards/length_penalty/std": 0.33426427841186523, "sampling/importance_sampling_ratio/max": 2.498494863510132, "sampling/importance_sampling_ratio/mean": 0.9682924151420593, "sampling/importance_sampling_ratio/min": 0.21204248070716858, "sampling/sampling_logp_difference/max": 1.5509686470031738, "sampling/sampling_logp_difference/mean": 0.05778278410434723, "step": 265, "step_time": 29.669615261955187 }, { "clip_ratio/high_max": 0.00039006458246149123, "clip_ratio/high_mean": 0.00039006458246149123, "clip_ratio/low_mean": 0.0004190996017617484, "clip_ratio/low_min": 0.0004190996017617484, "clip_ratio/region_mean": 0.0008091641939245164, "completions/clipped_ratio": 0.8166667222976685, "completions/max_length": 2048.0, "completions/max_terminated_length": 1112.0, "completions/mean_length": 1780.9500732421875, "completions/mean_terminated_length": 591.3636474609375, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 4.914393345514934, "epoch": 0.3877551020408163, "frac_reward_zero_std": 0.0, "grad_norm": 0.06251244246959686, "learning_rate": 2e-05, "loss": 0.1071762666106224, "num_tokens": 15647117.0, "reward": -0.769604504108429, "reward_std": 0.4822118282318115, "rewards/correctness/mean": 0.10000000149011612, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.8696044683456421, "rewards/length_penalty/std": 0.2825454771518707, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9601603746414185, "sampling/importance_sampling_ratio/min": 0.2065960019826889, "sampling/sampling_logp_difference/max": 1.5769901275634766, "sampling/sampling_logp_difference/mean": 0.07228940725326538, "step": 266, "step_time": 31.469009269494563 }, { "clip_ratio/high_max": 0.00018256455708372718, "clip_ratio/high_mean": 0.00018256455708372718, "clip_ratio/low_mean": 0.0002890966085639472, "clip_ratio/low_min": 0.0002890966085639472, "clip_ratio/region_mean": 0.00047166116322235513, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 1487.0, "completions/mean_length": 1835.7000732421875, "completions/mean_terminated_length": 632.6666870117188, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 4.509005069732666, "epoch": 0.3892128279883382, "frac_reward_zero_std": 0.0, "grad_norm": 0.07111051678657532, "learning_rate": 2e-05, "loss": 0.14451773464679718, "num_tokens": 15763669.0, "reward": -0.7463379502296448, "reward_std": 0.5907199382781982, "rewards/correctness/mean": 0.15000000596046448, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.8963378667831421, "rewards/length_penalty/std": 0.26398465037345886, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9625834226608276, "sampling/importance_sampling_ratio/min": 0.0025720433332026005, "sampling/sampling_logp_difference/max": 5.963054656982422, "sampling/sampling_logp_difference/mean": 0.06979194283485413, "step": 267, "step_time": 31.21758706565015 }, { "clip_ratio/high_max": 0.0006865921701925496, "clip_ratio/high_mean": 0.0006865921701925496, "clip_ratio/low_mean": 0.00041071732024041313, "clip_ratio/low_min": 0.00041071732024041313, "clip_ratio/region_mean": 0.0010973094710304092, "completions/clipped_ratio": 0.9166666865348816, "completions/max_length": 2048.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 1924.2000732421875, "completions/mean_terminated_length": 562.4000244140625, "completions/min_length": 485.0, "completions/min_terminated_length": 485.0, "entropy": 5.7037568887074785, "epoch": 0.39067055393586003, "frac_reward_zero_std": 0.0, "grad_norm": 0.04891866445541382, "learning_rate": 2e-05, "loss": 0.07960878312587738, "num_tokens": 15883621.0, "reward": -0.9228841662406921, "reward_std": 0.2854467034339905, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9395507574081421, "rewards/length_penalty/std": 0.20274989306926727, "sampling/importance_sampling_ratio/max": 2.143449306488037, "sampling/importance_sampling_ratio/mean": 0.9530673027038574, "sampling/importance_sampling_ratio/min": 0.005816253367811441, "sampling/sampling_logp_difference/max": 5.147099018096924, "sampling/sampling_logp_difference/mean": 0.08714986592531204, "step": 268, "step_time": 31.815797716146335 }, { "clip_ratio/high_max": 8.198669759925299e-05, "clip_ratio/high_mean": 8.198669759925299e-05, "clip_ratio/low_mean": 0.0004847698607287991, "clip_ratio/low_min": 0.0004847698607287991, "clip_ratio/region_mean": 0.0005667565589343818, "completions/clipped_ratio": 0.9333333969116211, "completions/max_length": 2048.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 1956.9500732421875, "completions/mean_terminated_length": 682.25, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 5.990203857421875, "epoch": 0.39212827988338195, "frac_reward_zero_std": 0.0, "grad_norm": 0.0793255940079689, "learning_rate": 2e-05, "loss": 0.13124756515026093, "num_tokens": 16005828.0, "reward": -0.9555420279502869, "reward_std": 0.17081841826438904, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.9555419683456421, "rewards/length_penalty/std": 0.17081841826438904, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9521302580833435, "sampling/importance_sampling_ratio/min": 0.05192401260137558, "sampling/sampling_logp_difference/max": 2.9579739570617676, "sampling/sampling_logp_difference/mean": 0.09166358411312103, "step": 269, "step_time": 31.38484827778302 }, { "clip_ratio/high_max": 2.7291446410041924e-05, "clip_ratio/high_mean": 2.7291446410041924e-05, "clip_ratio/low_mean": 0.00014517978039899995, "clip_ratio/low_min": 0.00014517978039899995, "clip_ratio/region_mean": 0.00017247122013941407, "completions/clipped_ratio": 0.9333333969116211, "completions/max_length": 2048.0, "completions/max_terminated_length": 625.0, "completions/mean_length": 1942.166748046875, "completions/mean_terminated_length": 460.5, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 7.195971171061198, "epoch": 0.3935860058309038, "frac_reward_zero_std": 0.0, "grad_norm": 0.0982329472899437, "learning_rate": 2e-05, "loss": 0.1663159430027008, "num_tokens": 16129378.0, "reward": -0.8816569447517395, "reward_std": 0.44692182540893555, "rewards/correctness/mean": 0.06666667014360428, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.9483235478401184, "rewards/length_penalty/std": 0.19586901366710663, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9457525610923767, "sampling/importance_sampling_ratio/min": 0.022035447880625725, "sampling/sampling_logp_difference/max": 3.8151028156280518, "sampling/sampling_logp_difference/mean": 0.10677551478147507, "step": 270, "step_time": 33.31503523304127 }, { "clip_ratio/high_max": 0.00020577786926878616, "clip_ratio/high_mean": 0.00020577786926878616, "clip_ratio/low_mean": 0.00028730503739401075, "clip_ratio/low_min": 0.00028730503739401075, "clip_ratio/region_mean": 0.000493082906662797, "completions/clipped_ratio": 0.9333333969116211, "completions/max_length": 2048.0, "completions/max_terminated_length": 1200.0, "completions/mean_length": 1956.050048828125, "completions/mean_terminated_length": 668.75, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 7.518760601679484, "epoch": 0.39504373177842567, "frac_reward_zero_std": 0.0, "grad_norm": 0.11048030853271484, "learning_rate": 2e-05, "loss": 0.08261695504188538, "num_tokens": 16253191.0, "reward": -0.8384358882904053, "reward_std": 0.4366929829120636, "rewards/correctness/mean": 0.11666666716337204, "rewards/correctness/std": 0.32373178005218506, "rewards/length_penalty/mean": -0.9551025629043579, "rewards/length_penalty/std": 0.17716452479362488, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9426504373550415, "sampling/importance_sampling_ratio/min": 0.10559508204460144, "sampling/sampling_logp_difference/max": 2.248143434524536, "sampling/sampling_logp_difference/mean": 0.11544980108737946, "step": 271, "step_time": 33.06732590892352 }, { "clip_ratio/high_max": 0.001102153551376735, "clip_ratio/high_mean": 0.001102153551376735, "clip_ratio/low_mean": 8.053317045172055e-05, "clip_ratio/low_min": 8.053317045172055e-05, "clip_ratio/region_mean": 0.0011826867315297325, "completions/clipped_ratio": 0.8833333849906921, "completions/max_length": 2048.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 1869.6168212890625, "completions/mean_terminated_length": 519.0, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 7.9727857907613116, "epoch": 0.3965014577259475, "frac_reward_zero_std": 0.0, "grad_norm": 0.12935522198677063, "learning_rate": 2e-05, "loss": 0.15854457020759583, "num_tokens": 16369908.0, "reward": -0.8128988146781921, "reward_std": 0.5463635325431824, "rewards/correctness/mean": 0.10000000149011612, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.9128987789154053, "rewards/length_penalty/std": 0.2476273626089096, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9411842226982117, "sampling/importance_sampling_ratio/min": 0.015444685705006123, "sampling/sampling_logp_difference/max": 4.170490264892578, "sampling/sampling_logp_difference/mean": 0.12215010076761246, "step": 272, "step_time": 31.090671005193144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 8.352002461751303, "epoch": 0.3979591836734694, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 16498458.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9378823637962341, "sampling/importance_sampling_ratio/min": 0.01892770081758499, "sampling/sampling_logp_difference/max": 3.9671287536621094, "sampling/sampling_logp_difference/mean": 0.12917236983776093, "step": 273, "step_time": 32.82281235139817 }, { "clip_ratio/high_max": 8.900961499117936e-05, "clip_ratio/high_mean": 8.900961499117936e-05, "clip_ratio/low_mean": 6.459100586653221e-05, "clip_ratio/low_min": 6.459100586653221e-05, "clip_ratio/region_mean": 0.00015360062085771156, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 570.0, "completions/mean_length": 1798.800048828125, "completions/mean_terminated_length": 386.6666564941406, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 8.618472576141357, "epoch": 0.39941690962099125, "frac_reward_zero_std": 0.0, "grad_norm": 0.09421354532241821, "learning_rate": 2e-05, "loss": 0.11156456172466278, "num_tokens": 16610216.0, "reward": -0.7783203721046448, "reward_std": 0.5653495192527771, "rewards/correctness/mean": 0.10000000149011612, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.8783203363418579, "rewards/length_penalty/std": 0.29246610403060913, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9367275238037109, "sampling/importance_sampling_ratio/min": 0.03733813762664795, "sampling/sampling_logp_difference/max": 3.2877399921417236, "sampling/sampling_logp_difference/mean": 0.14176608622074127, "step": 274, "step_time": 30.61649797903374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 8.595058759053549, "epoch": 0.4008746355685131, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 16738546.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9364311695098877, "sampling/importance_sampling_ratio/min": 0.04340706020593643, "sampling/sampling_logp_difference/max": 3.1371331214904785, "sampling/sampling_logp_difference/mean": 0.14039649069309235, "step": 275, "step_time": 32.78924954729155 }, { "clip_ratio/high_max": 4.3979349356959574e-05, "clip_ratio/high_mean": 4.3979349356959574e-05, "clip_ratio/low_mean": 0.00022536225757600428, "clip_ratio/low_min": 0.00022536225757600428, "clip_ratio/region_mean": 0.0002693416087519533, "completions/clipped_ratio": 0.9666666984558105, "completions/max_length": 2048.0, "completions/max_terminated_length": 671.0, "completions/mean_length": 1998.8834228515625, "completions/mean_terminated_length": 574.5, "completions/min_length": 478.0, "completions/min_terminated_length": 478.0, "entropy": 9.515000820159912, "epoch": 0.40233236151603496, "frac_reward_zero_std": 0.0, "grad_norm": 0.09098364412784576, "learning_rate": 2e-05, "loss": 0.08573106676340103, "num_tokens": 16864839.0, "reward": -0.9593506455421448, "reward_std": 0.2426481992006302, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9760172367095947, "rewards/length_penalty/std": 0.13052955269813538, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9319080114364624, "sampling/importance_sampling_ratio/min": 0.01979968510568142, "sampling/sampling_logp_difference/max": 3.9220893383026123, "sampling/sampling_logp_difference/mean": 0.15311644971370697, "step": 276, "step_time": 33.411174670793116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 9.675850550333658, "epoch": 0.4037900874635568, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 16992849.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9323450922966003, "sampling/importance_sampling_ratio/min": 0.006212851498275995, "sampling/sampling_logp_difference/max": 5.081135272979736, "sampling/sampling_logp_difference/mean": 0.15738143026828766, "step": 277, "step_time": 31.83429768285714 }, { "clip_ratio/high_max": 0.0009365811856696382, "clip_ratio/high_mean": 0.0009365811856696382, "clip_ratio/low_mean": 8.92697729189725e-06, "clip_ratio/low_min": 8.92697729189725e-06, "clip_ratio/region_mean": 0.0009455081647805249, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 2048.0, "completions/max_terminated_length": 401.0, "completions/mean_length": 1872.3668212890625, "completions/mean_terminated_length": 291.66668701171875, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 9.745427131652832, "epoch": 0.40524781341107874, "frac_reward_zero_std": 0.0, "grad_norm": 0.12919986248016357, "learning_rate": 2e-05, "loss": 0.1574300229549408, "num_tokens": 17111891.0, "reward": -0.8142415881156921, "reward_std": 0.5620673298835754, "rewards/correctness/mean": 0.10000000149011612, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.9142415523529053, "rewards/length_penalty/std": 0.2596394717693329, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9334349632263184, "sampling/importance_sampling_ratio/min": 0.05878634750843048, "sampling/sampling_logp_difference/max": 2.833845615386963, "sampling/sampling_logp_difference/mean": 0.1578126698732376, "step": 278, "step_time": 32.672706603305414 }, { "clip_ratio/high_max": 0.0007552358826311926, "clip_ratio/high_mean": 0.0007552358826311926, "clip_ratio/low_mean": 3.3367416108376347e-05, "clip_ratio/low_min": 3.3367416108376347e-05, "clip_ratio/region_mean": 0.0007886032981332392, "completions/clipped_ratio": 0.98333340883255, "completions/max_length": 2048.0, "completions/max_terminated_length": 183.0, "completions/mean_length": 2016.916748046875, "completions/mean_terminated_length": 183.0, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 10.387105782826742, "epoch": 0.4067055393586006, "frac_reward_zero_std": 0.0, "grad_norm": 0.07462750375270844, "learning_rate": 2e-05, "loss": 0.10269042104482651, "num_tokens": 17237096.0, "reward": -0.9681559801101685, "reward_std": 0.2466631531715393, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9848225712776184, "rewards/length_penalty/std": 0.11756370961666107, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9352046251296997, "sampling/importance_sampling_ratio/min": 0.030590863898396492, "sampling/sampling_logp_difference/max": 3.487053871154785, "sampling/sampling_logp_difference/mean": 0.1527576446533203, "step": 279, "step_time": 31.45041149482131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 10.560407002766928, "epoch": 0.40816326530612246, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 17364866.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.939231276512146, "sampling/importance_sampling_ratio/min": 7.990825542947277e-05, "sampling/sampling_logp_difference/max": 9.43463134765625, "sampling/sampling_logp_difference/mean": 0.13741585612297058, "step": 280, "step_time": 32.56683851382695 }, { "clip_ratio/high_max": 1.7832940405545134e-05, "clip_ratio/high_mean": 1.7832940405545134e-05, "clip_ratio/low_mean": 4.069010477299647e-05, "clip_ratio/low_min": 4.069010477299647e-05, "clip_ratio/region_mean": 5.85230451785416e-05, "completions/clipped_ratio": 0.98333340883255, "completions/max_length": 2048.0, "completions/max_terminated_length": 260.0, "completions/mean_length": 2018.2000732421875, "completions/mean_terminated_length": 260.0, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 10.637446721394857, "epoch": 0.4096209912536443, "frac_reward_zero_std": 0.0, "grad_norm": 0.0478152297437191, "learning_rate": 2e-05, "loss": 0.09919631481170654, "num_tokens": 17489788.0, "reward": -0.9687826037406921, "reward_std": 0.24180930852890015, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9854491949081421, "rewards/length_penalty/std": 0.11270986497402191, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9422204494476318, "sampling/importance_sampling_ratio/min": 0.004846750758588314, "sampling/sampling_logp_difference/max": 5.329446792602539, "sampling/sampling_logp_difference/mean": 0.12537381052970886, "step": 281, "step_time": 32.08853977639228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 10.760574499766031, "epoch": 0.4110787172011662, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 17618338.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9436192512512207, "sampling/importance_sampling_ratio/min": 0.024097217246890068, "sampling/sampling_logp_difference/max": 3.725658893585205, "sampling/sampling_logp_difference/mean": 0.11478592455387115, "step": 282, "step_time": 31.93326418218203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 10.889357725779215, "epoch": 0.41253644314868804, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 17746398.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9451031684875488, "sampling/importance_sampling_ratio/min": 0.020302874967455864, "sampling/sampling_logp_difference/max": 3.8969926834106445, "sampling/sampling_logp_difference/mean": 0.10222256928682327, "step": 283, "step_time": 32.83494277112186 }, { "clip_ratio/high_max": 6.560388525637488e-05, "clip_ratio/high_mean": 6.560388525637488e-05, "clip_ratio/low_mean": 6.529156053147744e-05, "clip_ratio/low_min": 6.529156053147744e-05, "clip_ratio/region_mean": 0.00013089544821317153, "completions/clipped_ratio": 0.9666666984558105, "completions/max_length": 2048.0, "completions/max_terminated_length": 1892.0, "completions/mean_length": 2018.1168212890625, "completions/mean_terminated_length": 1151.5, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "entropy": 10.560033798217773, "epoch": 0.4139941690962099, "frac_reward_zero_std": 0.0, "grad_norm": 0.08786778897047043, "learning_rate": 2e-05, "loss": 0.07074017822742462, "num_tokens": 17871065.0, "reward": -0.9520752429962158, "reward_std": 0.2686399519443512, "rewards/correctness/mean": 0.03333333507180214, "rewards/correctness/std": 0.1810203194618225, "rewards/length_penalty/mean": -0.9854085445404053, "rewards/length_penalty/std": 0.10349274426698685, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9464466571807861, "sampling/importance_sampling_ratio/min": 0.0006484799669124186, "sampling/sampling_logp_difference/max": 7.340879440307617, "sampling/sampling_logp_difference/mean": 0.0954880639910698, "step": 284, "step_time": 32.11458393349312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 10.91657559076945, "epoch": 0.41545189504373176, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 17997625.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9459961652755737, "sampling/importance_sampling_ratio/min": 0.0005569281056523323, "sampling/sampling_logp_difference/max": 7.493074417114258, "sampling/sampling_logp_difference/mean": 0.09484820812940598, "step": 285, "step_time": 31.35973776690662 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 10.876597881317139, "epoch": 0.41690962099125367, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 18126025.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9468587636947632, "sampling/importance_sampling_ratio/min": 0.024051105603575706, "sampling/sampling_logp_difference/max": 3.727574348449707, "sampling/sampling_logp_difference/mean": 0.09144081920385361, "step": 286, "step_time": 32.99028211622499 }, { "clip_ratio/high_max": 0.00021042504886281677, "clip_ratio/high_mean": 0.00021042504886281677, "clip_ratio/low_mean": 3.329395318966514e-05, "clip_ratio/low_min": 3.329395318966514e-05, "clip_ratio/region_mean": 0.00024371900265881172, "completions/clipped_ratio": 0.9666666984558105, "completions/max_length": 2048.0, "completions/max_terminated_length": 337.0, "completions/mean_length": 1988.3668212890625, "completions/mean_terminated_length": 259.0, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 11.003250281016031, "epoch": 0.41836734693877553, "frac_reward_zero_std": 0.0, "grad_norm": 0.04854585602879524, "learning_rate": 2e-05, "loss": 0.09593209624290466, "num_tokens": 18248867.0, "reward": -0.9542155265808105, "reward_std": 0.26278021931648254, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9708821773529053, "rewards/length_penalty/std": 0.1582830250263214, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9469846487045288, "sampling/importance_sampling_ratio/min": 1.0366252354288008e-06, "sampling/sampling_logp_difference/max": 13.779540061950684, "sampling/sampling_logp_difference/mean": 0.09163237363100052, "step": 287, "step_time": 31.3422354941722 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 11.130181471506754, "epoch": 0.4198250728862974, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 18376117.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9461483359336853, "sampling/importance_sampling_ratio/min": 0.022621961310505867, "sampling/sampling_logp_difference/max": 3.7888340950012207, "sampling/sampling_logp_difference/mean": 0.09254187345504761, "step": 288, "step_time": 32.218342249048874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 11.183223565419516, "epoch": 0.42128279883381925, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 18502467.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9458378553390503, "sampling/importance_sampling_ratio/min": 0.01253539603203535, "sampling/sampling_logp_difference/max": 4.379199028015137, "sampling/sampling_logp_difference/mean": 0.09579569846391678, "step": 289, "step_time": 32.19404075015336 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.2915243587534256e-05, "clip_ratio/low_min": 4.2915243587534256e-05, "clip_ratio/region_mean": 4.2915243587534256e-05, "completions/clipped_ratio": 0.9666666984558105, "completions/max_length": 2048.0, "completions/max_terminated_length": 342.0, "completions/mean_length": 1990.900146484375, "completions/mean_terminated_length": 335.0, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 11.117544809977213, "epoch": 0.4227405247813411, "frac_reward_zero_std": 0.0, "grad_norm": 0.06452081352472305, "learning_rate": 2e-05, "loss": 0.12746074795722961, "num_tokens": 18627671.0, "reward": -0.9387858510017395, "reward_std": 0.33243098855018616, "rewards/correctness/mean": 0.03333333507180214, "rewards/correctness/std": 0.1810203194618225, "rewards/length_penalty/mean": -0.972119152545929, "rewards/length_penalty/std": 0.15141138434410095, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.946613073348999, "sampling/importance_sampling_ratio/min": 0.026754189282655716, "sampling/sampling_logp_difference/max": 3.6210641860961914, "sampling/sampling_logp_difference/mean": 0.0984814241528511, "step": 290, "step_time": 31.71451087691821 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 11.281938076019287, "epoch": 0.42419825072886297, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 18756981.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9462643265724182, "sampling/importance_sampling_ratio/min": 0.0031906976364552975, "sampling/sampling_logp_difference/max": 5.747515678405762, "sampling/sampling_logp_difference/mean": 0.10182085633277893, "step": 291, "step_time": 32.77314696600661 }, { "clip_ratio/high_max": 4.313610043027438e-05, "clip_ratio/high_mean": 4.313610043027438e-05, "clip_ratio/low_mean": 0.00020903432838773975, "clip_ratio/low_min": 0.00020903432838773975, "clip_ratio/region_mean": 0.00025217042639269494, "completions/clipped_ratio": 0.9666666984558105, "completions/max_length": 2048.0, "completions/max_terminated_length": 269.0, "completions/mean_length": 1987.2667236328125, "completions/mean_terminated_length": 226.0, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 11.04299259185791, "epoch": 0.42565597667638483, "frac_reward_zero_std": 0.0, "grad_norm": 0.062462370842695236, "learning_rate": 2e-05, "loss": 0.09625676274299622, "num_tokens": 18879917.0, "reward": -0.92034512758255, "reward_std": 0.3627047836780548, "rewards/correctness/mean": 0.05000000074505806, "rewards/correctness/std": 0.2197841852903366, "rewards/length_penalty/mean": -0.9703450798988342, "rewards/length_penalty/std": 0.16109085083007812, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9467905163764954, "sampling/importance_sampling_ratio/min": 0.007742264308035374, "sampling/sampling_logp_difference/max": 4.861061096191406, "sampling/sampling_logp_difference/mean": 0.10382029414176941, "step": 292, "step_time": 31.848492444958538 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 11.249570687611898, "epoch": 0.4271137026239067, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 19006927.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9472648501396179, "sampling/importance_sampling_ratio/min": 0.015321177430450916, "sampling/sampling_logp_difference/max": 4.178519248962402, "sampling/sampling_logp_difference/mean": 0.10690370202064514, "step": 293, "step_time": 31.400095215067267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 11.214203675587973, "epoch": 0.42857142857142855, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 19136137.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9474822878837585, "sampling/importance_sampling_ratio/min": 0.005025867838412523, "sampling/sampling_logp_difference/max": 5.29315710067749, "sampling/sampling_logp_difference/mean": 0.1083599254488945, "step": 294, "step_time": 32.700320662232116 }, { "clip_ratio/high_max": 8.870438250596635e-06, "clip_ratio/high_mean": 8.870438250596635e-06, "clip_ratio/low_mean": 1.6276041909198586e-05, "clip_ratio/low_min": 1.6276041909198586e-05, "clip_ratio/region_mean": 2.514648015979522e-05, "completions/clipped_ratio": 0.98333340883255, "completions/max_length": 2048.0, "completions/max_terminated_length": 357.0, "completions/mean_length": 2019.8167724609375, "completions/mean_terminated_length": 357.0, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 11.240072250366211, "epoch": 0.43002915451895046, "frac_reward_zero_std": 0.0, "grad_norm": 0.04093112424015999, "learning_rate": 2e-05, "loss": 0.0942196175456047, "num_tokens": 19266256.0, "reward": -0.9695720076560974, "reward_std": 0.2356947362422943, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9862385988235474, "rewards/length_penalty/std": 0.10659529268741608, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9484888315200806, "sampling/importance_sampling_ratio/min": 2.2983890630712267e-06, "sampling/sampling_logp_difference/max": 12.983302116394043, "sampling/sampling_logp_difference/mean": 0.11081989109516144, "step": 295, "step_time": 34.177097915438935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.897972525621299e-05, "clip_ratio/low_min": 6.897972525621299e-05, "clip_ratio/region_mean": 6.897972525621299e-05, "completions/clipped_ratio": 0.98333340883255, "completions/max_length": 2048.0, "completions/max_terminated_length": 267.0, "completions/mean_length": 2018.3167724609375, "completions/mean_terminated_length": 267.0, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 11.149376392364502, "epoch": 0.4314868804664723, "frac_reward_zero_std": 0.0, "grad_norm": 0.06764543801546097, "learning_rate": 2e-05, "loss": 0.07088764756917953, "num_tokens": 19392255.0, "reward": -0.9521728754043579, "reward_std": 0.2717883884906769, "rewards/correctness/mean": 0.03333333507180214, "rewards/correctness/std": 0.1810203194618225, "rewards/length_penalty/mean": -0.9855061769485474, "rewards/length_penalty/std": 0.1122686043381691, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9483795166015625, "sampling/importance_sampling_ratio/min": 0.024467486888170242, "sampling/sampling_logp_difference/max": 3.7104101181030273, "sampling/sampling_logp_difference/mean": 0.1120859757065773, "step": 296, "step_time": 32.23613230395131 }, { "clip_ratio/high_max": 0.00029774268841720186, "clip_ratio/high_mean": 0.00029774268841720186, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00029774268841720186, "completions/clipped_ratio": 0.98333340883255, "completions/max_length": 2048.0, "completions/max_terminated_length": 224.0, "completions/mean_length": 2017.60009765625, "completions/mean_terminated_length": 224.0, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 11.147947470347086, "epoch": 0.4329446064139942, "frac_reward_zero_std": 0.0, "grad_norm": 0.036026909947395325, "learning_rate": 2e-05, "loss": 0.09742555022239685, "num_tokens": 19519081.0, "reward": -0.9851562976837158, "reward_std": 0.11497919261455536, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.985156238079071, "rewards/length_penalty/std": 0.11497919261455536, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.948697566986084, "sampling/importance_sampling_ratio/min": 0.015425711870193481, "sampling/sampling_logp_difference/max": 4.171719551086426, "sampling/sampling_logp_difference/mean": 0.10924047231674194, "step": 297, "step_time": 32.50894394819625 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013140809096512385, "clip_ratio/low_min": 0.00013140809096512385, "clip_ratio/region_mean": 0.00013140809096512385, "completions/clipped_ratio": 0.98333340883255, "completions/max_length": 2048.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 2024.5667724609375, "completions/mean_terminated_length": 642.0, "completions/min_length": 642.0, "completions/min_terminated_length": 642.0, "entropy": 11.24603303273519, "epoch": 0.43440233236151604, "frac_reward_zero_std": 0.0, "grad_norm": 0.04536767676472664, "learning_rate": 2e-05, "loss": 0.07472822070121765, "num_tokens": 19645535.0, "reward": -0.9885579943656921, "reward_std": 0.08862978965044022, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.9885579347610474, "rewards/length_penalty/std": 0.08862979710102081, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9487028121948242, "sampling/importance_sampling_ratio/min": 0.008879927918314934, "sampling/sampling_logp_difference/max": 4.72396183013916, "sampling/sampling_logp_difference/mean": 0.10927778482437134, "step": 298, "step_time": 31.84186882013455 }, { "clip_ratio/high_max": 1.8906732293544337e-05, "clip_ratio/high_mean": 1.8906732293544337e-05, "clip_ratio/low_mean": 0.00016143258168691924, "clip_ratio/low_min": 0.00016143258168691924, "clip_ratio/region_mean": 0.00018033931155514438, "completions/clipped_ratio": 0.9333333969116211, "completions/max_length": 2048.0, "completions/max_terminated_length": 220.0, "completions/mean_length": 1924.2501220703125, "completions/mean_terminated_length": 191.75, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "entropy": 11.07070795694987, "epoch": 0.4358600583090379, "frac_reward_zero_std": 0.0, "grad_norm": 0.05931065231561661, "learning_rate": 2e-05, "loss": 0.16998939216136932, "num_tokens": 19765990.0, "reward": -0.8729085922241211, "reward_std": 0.47955334186553955, "rewards/correctness/mean": 0.06666667014360428, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.9395751953125, "rewards/length_penalty/std": 0.2280127853155136, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.949758768081665, "sampling/importance_sampling_ratio/min": 0.013378970324993134, "sampling/sampling_logp_difference/max": 4.314071178436279, "sampling/sampling_logp_difference/mean": 0.10347594320774078, "step": 299, "step_time": 32.08467884873971 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 11.32125727335612, "epoch": 0.43731778425655976, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 19894780.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9486837983131409, "sampling/importance_sampling_ratio/min": 0.024938564747571945, "sampling/sampling_logp_difference/max": 3.6913399696350098, "sampling/sampling_logp_difference/mean": 0.10162407159805298, "step": 300, "step_time": 32.008629967225716 }, { "clip_ratio/high_max": 8.928411868206846e-06, "clip_ratio/high_mean": 8.928411868206846e-06, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.928411868206846e-06, "completions/clipped_ratio": 0.98333340883255, "completions/max_length": 2048.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 2017.783447265625, "completions/mean_terminated_length": 235.0, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 11.284760475158691, "epoch": 0.4387755102040816, "frac_reward_zero_std": 0.0, "grad_norm": 0.03290018066763878, "learning_rate": 2e-05, "loss": 0.10140568763017654, "num_tokens": 20020787.0, "reward": -0.9685791730880737, "reward_std": 0.2433852255344391, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9852457642555237, "rewards/length_penalty/std": 0.11428578943014145, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9486204981803894, "sampling/importance_sampling_ratio/min": 0.010691052302718163, "sampling/sampling_logp_difference/max": 4.538348197937012, "sampling/sampling_logp_difference/mean": 0.09953093528747559, "step": 301, "step_time": 32.444805392064154 }, { "clip_ratio/high_max": 2.6600173441693187e-05, "clip_ratio/high_mean": 2.6600173441693187e-05, "clip_ratio/low_mean": 5.10638583364198e-05, "clip_ratio/low_min": 5.10638583364198e-05, "clip_ratio/region_mean": 7.766403177811299e-05, "completions/clipped_ratio": 0.9666666984558105, "completions/max_length": 2048.0, "completions/max_terminated_length": 400.0, "completions/mean_length": 1991.3167724609375, "completions/mean_terminated_length": 347.5, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 11.304510275522867, "epoch": 0.4402332361516035, "frac_reward_zero_std": 0.0, "grad_norm": 0.033597152680158615, "learning_rate": 2e-05, "loss": 0.0982612669467926, "num_tokens": 20145796.0, "reward": -0.9556559920310974, "reward_std": 0.25953397154808044, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9723225831985474, "rewards/length_penalty/std": 0.15037930011749268, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9488328695297241, "sampling/importance_sampling_ratio/min": 0.018843168392777443, "sampling/sampling_logp_difference/max": 3.971604824066162, "sampling/sampling_logp_difference/mean": 0.09697035700082779, "step": 302, "step_time": 32.53830000711605 }, { "clip_ratio/high_max": 0.0002419021917982415, "clip_ratio/high_mean": 0.0002419021917982415, "clip_ratio/low_mean": 5.0624340777479425e-05, "clip_ratio/low_min": 5.0624340777479425e-05, "clip_ratio/region_mean": 0.0002925265301504017, "completions/clipped_ratio": 0.9333333969116211, "completions/max_length": 2048.0, "completions/max_terminated_length": 258.0, "completions/mean_length": 1926.1334228515625, "completions/mean_terminated_length": 220.0, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 11.057292620340982, "epoch": 0.44169096209912534, "frac_reward_zero_std": 0.0, "grad_norm": 0.05433151498436928, "learning_rate": 2e-05, "loss": 0.15714368224143982, "num_tokens": 20267164.0, "reward": -0.8904948234558105, "reward_std": 0.42848503589630127, "rewards/correctness/mean": 0.05000000074505806, "rewards/correctness/std": 0.2197841852903366, "rewards/length_penalty/mean": -0.9404947757720947, "rewards/length_penalty/std": 0.2245575338602066, "sampling/importance_sampling_ratio/max": 2.648115634918213, "sampling/importance_sampling_ratio/mean": 0.9491006135940552, "sampling/importance_sampling_ratio/min": 0.003927208483219147, "sampling/sampling_logp_difference/max": 5.539826393127441, "sampling/sampling_logp_difference/mean": 0.09296528995037079, "step": 303, "step_time": 31.468342908890918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 11.389177958170572, "epoch": 0.44314868804664725, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 20394944.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9487767815589905, "sampling/importance_sampling_ratio/min": 0.013516728766262531, "sampling/sampling_logp_difference/max": 4.303827285766602, "sampling/sampling_logp_difference/mean": 0.09210412204265594, "step": 304, "step_time": 32.356076672207564 }, { "clip_ratio/high_max": 1.790478199836798e-05, "clip_ratio/high_mean": 1.790478199836798e-05, "clip_ratio/low_mean": 2.5911554378884222e-05, "clip_ratio/low_min": 2.5911554378884222e-05, "clip_ratio/region_mean": 4.38163357709224e-05, "completions/clipped_ratio": 0.9333333969116211, "completions/max_length": 2048.0, "completions/max_terminated_length": 462.0, "completions/mean_length": 1935.2000732421875, "completions/mean_terminated_length": 356.0, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 11.302481174468994, "epoch": 0.4446064139941691, "frac_reward_zero_std": 0.0, "grad_norm": 0.035306062549352646, "learning_rate": 2e-05, "loss": 0.12357667833566666, "num_tokens": 20514906.0, "reward": -0.9115886092185974, "reward_std": 0.3606376349925995, "rewards/correctness/mean": 0.03333333507180214, "rewards/correctness/std": 0.1810203194618225, "rewards/length_penalty/mean": -0.9449218511581421, "rewards/length_penalty/std": 0.20826652646064758, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9488691091537476, "sampling/importance_sampling_ratio/min": 0.026045838370919228, "sampling/sampling_logp_difference/max": 3.647897243499756, "sampling/sampling_logp_difference/mean": 0.089111328125, "step": 305, "step_time": 31.13226025737822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.138020954599293e-06, "clip_ratio/low_min": 8.138020954599293e-06, "clip_ratio/region_mean": 8.138020954599293e-06, "completions/clipped_ratio": 0.98333340883255, "completions/max_length": 2048.0, "completions/max_terminated_length": 227.0, "completions/mean_length": 2017.650146484375, "completions/mean_terminated_length": 227.0, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 11.346004645029703, "epoch": 0.446064139941691, "frac_reward_zero_std": 0.0, "grad_norm": 0.032005809247493744, "learning_rate": 2e-05, "loss": 0.10208320617675781, "num_tokens": 20639685.0, "reward": -0.9685140252113342, "reward_std": 0.24388952553272247, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.985180675983429, "rewards/length_penalty/std": 0.11479008942842484, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9491950869560242, "sampling/importance_sampling_ratio/min": 0.08257041126489639, "sampling/sampling_logp_difference/max": 2.4941039085388184, "sampling/sampling_logp_difference/mean": 0.08729733526706696, "step": 306, "step_time": 31.91764120873995 }, { "clip_ratio/high_max": 8.89173437220355e-06, "clip_ratio/high_mean": 8.89173437220355e-06, "clip_ratio/low_mean": 0.00010805541266260359, "clip_ratio/low_min": 0.00010805541266260359, "clip_ratio/region_mean": 0.00011694714703480713, "completions/clipped_ratio": 0.98333340883255, "completions/max_length": 2048.0, "completions/max_terminated_length": 312.0, "completions/mean_length": 2019.0667724609375, "completions/mean_terminated_length": 312.0, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 11.37714417775472, "epoch": 0.44752186588921283, "frac_reward_zero_std": 0.0, "grad_norm": 0.0331091471016407, "learning_rate": 2e-05, "loss": 0.0969465970993042, "num_tokens": 20765259.0, "reward": -0.9692057967185974, "reward_std": 0.23853138089179993, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9858723878860474, "rewards/length_penalty/std": 0.10943195223808289, "sampling/importance_sampling_ratio/max": 2.9611902236938477, "sampling/importance_sampling_ratio/mean": 0.9486574530601501, "sampling/importance_sampling_ratio/min": 0.028793415054678917, "sampling/sampling_logp_difference/max": 3.5476086139678955, "sampling/sampling_logp_difference/mean": 0.08530811965465546, "step": 307, "step_time": 32.00056678173132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016654044217527067, "clip_ratio/low_min": 0.00016654044217527067, "clip_ratio/region_mean": 0.00016654044217527067, "completions/clipped_ratio": 0.9666666984558105, "completions/max_length": 2048.0, "completions/max_terminated_length": 319.0, "completions/mean_length": 1989.3834228515625, "completions/mean_terminated_length": 289.5, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 11.361408710479736, "epoch": 0.4489795918367347, "frac_reward_zero_std": 0.0, "grad_norm": 0.0300514604896307, "learning_rate": 2e-05, "loss": 0.0897468701004982, "num_tokens": 20889752.0, "reward": -0.9547119736671448, "reward_std": 0.2635475993156433, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9713785648345947, "rewards/length_penalty/std": 0.15545439720153809, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9486801028251648, "sampling/importance_sampling_ratio/min": 0.018677830696105957, "sampling/sampling_logp_difference/max": 3.9804179668426514, "sampling/sampling_logp_difference/mean": 0.08451830595731735, "step": 308, "step_time": 31.435338189126924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 11.315851847330729, "epoch": 0.45043731778425655, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 2e-05, "loss": 0.0, "num_tokens": 21017412.0, "reward": -1.0, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9486898183822632, "sampling/importance_sampling_ratio/min": 0.08134376257658005, "sampling/sampling_logp_difference/max": 2.509071111679077, "sampling/sampling_logp_difference/mean": 0.0834798812866211, "step": 309, "step_time": 32.588759526843205 }, { "clip_ratio/high_max": 0.0001605101988388924, "clip_ratio/high_mean": 0.0001605101988388924, "clip_ratio/low_mean": 0.00010514878583004854, "clip_ratio/low_min": 0.00010514878583004854, "clip_ratio/region_mean": 0.0002656589834562813, "completions/clipped_ratio": 0.8833333849906921, "completions/max_length": 2048.0, "completions/max_terminated_length": 304.0, "completions/mean_length": 1839.5001220703125, "completions/mean_terminated_length": 260.8571472167969, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 11.240314483642578, "epoch": 0.4518950437317784, "frac_reward_zero_std": 0.0, "grad_norm": 0.04892507568001747, "learning_rate": 2e-05, "loss": 0.18980324268341064, "num_tokens": 21131502.0, "reward": -0.8315267562866211, "reward_std": 0.4981188178062439, "rewards/correctness/mean": 0.06666667014360428, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.898193359375, "rewards/length_penalty/std": 0.28256016969680786, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9495506882667542, "sampling/importance_sampling_ratio/min": 0.042339976876974106, "sampling/sampling_logp_difference/max": 3.1620235443115234, "sampling/sampling_logp_difference/mean": 0.0813635066151619, "step": 310, "step_time": 31.242469509597868 }, { "clip_ratio/high_max": 0.00027429323684676393, "clip_ratio/high_mean": 0.00027429323684676393, "clip_ratio/low_mean": 2.6059167794301175e-05, "clip_ratio/low_min": 2.6059167794301175e-05, "clip_ratio/region_mean": 0.0003003524046410651, "completions/clipped_ratio": 0.9500000476837158, "completions/max_length": 2048.0, "completions/max_terminated_length": 196.0, "completions/mean_length": 1954.5667724609375, "completions/mean_terminated_length": 179.33334350585938, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 11.396289348602295, "epoch": 0.45335276967930027, "frac_reward_zero_std": 0.0, "grad_norm": 0.04884692654013634, "learning_rate": 2e-05, "loss": 0.16008888185024261, "num_tokens": 21253336.0, "reward": -0.9043782949447632, "reward_std": 0.42032504081726074, "rewards/correctness/mean": 0.05000000074505806, "rewards/correctness/std": 0.21978417038917542, "rewards/length_penalty/mean": -0.9543782472610474, "rewards/length_penalty/std": 0.20054315030574799, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9492292404174805, "sampling/importance_sampling_ratio/min": 0.0025428151711821556, "sampling/sampling_logp_difference/max": 5.974483489990234, "sampling/sampling_logp_difference/mean": 0.08100900799036026, "step": 311, "step_time": 31.18853940581903 }, { "clip_ratio/high_max": 4.894357395338981e-05, "clip_ratio/high_mean": 4.894357395338981e-05, "clip_ratio/low_mean": 3.333769503418201e-05, "clip_ratio/low_min": 3.333769503418201e-05, "clip_ratio/region_mean": 8.228126898757182e-05, "completions/clipped_ratio": 0.9166666865348816, "completions/max_length": 2048.0, "completions/max_terminated_length": 438.0, "completions/mean_length": 1901.416748046875, "completions/mean_terminated_length": 289.0, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 11.285658200581869, "epoch": 0.45481049562682213, "frac_reward_zero_std": 0.0, "grad_norm": 0.053343020379543304, "learning_rate": 2e-05, "loss": 0.19996294379234314, "num_tokens": 21371601.0, "reward": -0.8450928330421448, "reward_std": 0.5182434916496277, "rewards/correctness/mean": 0.0833333358168602, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.9284260869026184, "rewards/length_penalty/std": 0.23968692123889923, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9492059350013733, "sampling/importance_sampling_ratio/min": 0.02282441221177578, "sampling/sampling_logp_difference/max": 3.7799246311187744, "sampling/sampling_logp_difference/mean": 0.08013840019702911, "step": 312, "step_time": 31.550020158989355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.951823110692203e-05, "clip_ratio/low_min": 8.951823110692203e-05, "clip_ratio/region_mean": 8.951823110692203e-05, "completions/clipped_ratio": 1.0, "completions/max_length": 2048.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 2048.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 2048.0, "completions/min_terminated_length": 0.0, "entropy": 11.46130402882894, "epoch": 0.45626822157434405, "frac_reward_zero_std": 0.0, "grad_norm": 0.06520382314920425, "learning_rate": 2e-05, "loss": -0.00043542683124542236, "num_tokens": 21501451.0, "reward": -0.98333340883255, "reward_std": 0.12909944355487823, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -1.0, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9486974477767944, "sampling/importance_sampling_ratio/min": 0.022274570539593697, "sampling/sampling_logp_difference/max": 3.804309606552124, "sampling/sampling_logp_difference/mean": 0.07903284579515457, "step": 313, "step_time": 32.26152233383618 }, { "clip_ratio/high_max": 0.00019679340463577924, "clip_ratio/high_mean": 0.00019679340463577924, "clip_ratio/low_mean": 1.6276041909198586e-05, "clip_ratio/low_min": 1.6276041909198586e-05, "clip_ratio/region_mean": 0.00021306944411965864, "completions/clipped_ratio": 0.98333340883255, "completions/max_length": 2048.0, "completions/max_terminated_length": 340.0, "completions/mean_length": 2019.533447265625, "completions/mean_terminated_length": 340.0, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 11.453712622324625, "epoch": 0.4577259475218659, "frac_reward_zero_std": 0.0, "grad_norm": 0.07525785267353058, "learning_rate": 2e-05, "loss": 0.00028640657546930015, "num_tokens": 21628363.0, "reward": -0.9361003041267395, "reward_std": 0.24183407425880432, "rewards/correctness/mean": 0.05000000074505806, "rewards/correctness/std": 0.2197841852903366, "rewards/length_penalty/mean": -0.9861002564430237, "rewards/length_penalty/std": 0.10766691714525223, "sampling/importance_sampling_ratio/max": 2.064592123031616, "sampling/importance_sampling_ratio/mean": 0.9486229419708252, "sampling/importance_sampling_ratio/min": 0.07415920495986938, "sampling/sampling_logp_difference/max": 2.601541042327881, "sampling/sampling_logp_difference/mean": 0.07762716710567474, "step": 314, "step_time": 32.569067504256964 }, { "clip_ratio/high_max": 0.00020859134868563464, "clip_ratio/high_mean": 0.00020859134868563464, "clip_ratio/low_mean": 4.313177851145156e-05, "clip_ratio/low_min": 4.313177851145156e-05, "clip_ratio/region_mean": 0.00025172312355910737, "completions/clipped_ratio": 0.9166666865348816, "completions/max_length": 2048.0, "completions/max_terminated_length": 276.0, "completions/mean_length": 1895.7000732421875, "completions/mean_terminated_length": 220.40000915527344, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 11.400930086771647, "epoch": 0.45918367346938777, "frac_reward_zero_std": 0.0, "grad_norm": 0.046633847057819366, "learning_rate": 2e-05, "loss": 0.20750261843204498, "num_tokens": 21745925.0, "reward": -0.8423014879226685, "reward_std": 0.5274609327316284, "rewards/correctness/mean": 0.0833333358168602, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.9256347417831421, "rewards/length_penalty/std": 0.24876569211483002, "sampling/importance_sampling_ratio/max": 2.1901683807373047, "sampling/importance_sampling_ratio/mean": 0.9489058256149292, "sampling/importance_sampling_ratio/min": 0.06274791061878204, "sampling/sampling_logp_difference/max": 2.768630027770996, "sampling/sampling_logp_difference/mean": 0.0762314423918724, "step": 315, "step_time": 31.03224169788882 }, { "clip_ratio/high_max": 0.00027773217637635145, "clip_ratio/high_mean": 0.00027773217637635145, "clip_ratio/low_mean": 2.5170623606148485e-05, "clip_ratio/low_min": 2.5170623606148485e-05, "clip_ratio/region_mean": 0.00030290279634452116, "completions/clipped_ratio": 0.9333333969116211, "completions/max_length": 2048.0, "completions/max_terminated_length": 349.0, "completions/mean_length": 1932.416748046875, "completions/mean_terminated_length": 314.25, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 11.342920303344727, "epoch": 0.4606413994169096, "frac_reward_zero_std": 0.0, "grad_norm": 0.04251934215426445, "learning_rate": 2e-05, "loss": 0.14802336692810059, "num_tokens": 21866970.0, "reward": -0.8935628533363342, "reward_std": 0.41699230670928955, "rewards/correctness/mean": 0.05000000074505806, "rewards/correctness/std": 0.2197841852903366, "rewards/length_penalty/mean": -0.9435628056526184, "rewards/length_penalty/std": 0.21298547089099884, "sampling/importance_sampling_ratio/max": 2.0017716884613037, "sampling/importance_sampling_ratio/mean": 0.9488850831985474, "sampling/importance_sampling_ratio/min": 0.034942202270030975, "sampling/sampling_logp_difference/max": 3.354059934616089, "sampling/sampling_logp_difference/mean": 0.07468743622303009, "step": 316, "step_time": 31.97772161033936 }, { "clip_ratio/high_max": 2.6927868627050582e-05, "clip_ratio/high_mean": 2.6927868627050582e-05, "clip_ratio/low_mean": 1.6276041909198586e-05, "clip_ratio/low_min": 1.6276041909198586e-05, "clip_ratio/region_mean": 4.320391053624917e-05, "completions/clipped_ratio": 0.9500000476837158, "completions/max_length": 2048.0, "completions/max_terminated_length": 1131.0, "completions/mean_length": 1981.1334228515625, "completions/mean_terminated_length": 710.6666870117188, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 11.287573655446371, "epoch": 0.4620991253644315, "frac_reward_zero_std": 0.0, "grad_norm": 0.032221511006355286, "learning_rate": 2e-05, "loss": 0.09651438146829605, "num_tokens": 21992828.0, "reward": -0.9506836533546448, "reward_std": 0.25644609332084656, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9673502445220947, "rewards/length_penalty/std": 0.1479337364435196, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9492385387420654, "sampling/importance_sampling_ratio/min": 0.027673380449414253, "sampling/sampling_logp_difference/max": 3.5872843265533447, "sampling/sampling_logp_difference/mean": 0.07299625128507614, "step": 317, "step_time": 33.33431696589105 }, { "clip_ratio/high_max": 4.8490373349826164e-05, "clip_ratio/high_mean": 4.8490373349826164e-05, "clip_ratio/low_mean": 0.00018213820779540887, "clip_ratio/low_min": 0.00018213820779540887, "clip_ratio/region_mean": 0.00023062857871991582, "completions/clipped_ratio": 0.9000000357627869, "completions/max_length": 2048.0, "completions/max_terminated_length": 351.0, "completions/mean_length": 1867.6168212890625, "completions/mean_terminated_length": 244.1666717529297, "completions/min_length": 170.0, "completions/min_terminated_length": 170.0, "entropy": 11.291426022847494, "epoch": 0.46355685131195334, "frac_reward_zero_std": 0.0, "grad_norm": 0.04703188315033913, "learning_rate": 2e-05, "loss": 0.22522103786468506, "num_tokens": 22108815.0, "reward": -0.8119222521781921, "reward_std": 0.5690566897392273, "rewards/correctness/mean": 0.10000000149011612, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.9119222164154053, "rewards/length_penalty/std": 0.26659518480300903, "sampling/importance_sampling_ratio/max": 2.3297274112701416, "sampling/importance_sampling_ratio/mean": 0.9487144351005554, "sampling/importance_sampling_ratio/min": 0.014096592552959919, "sampling/sampling_logp_difference/max": 4.26182222366333, "sampling/sampling_logp_difference/mean": 0.0718219131231308, "step": 318, "step_time": 30.78137767291628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 3.255208442472698e-05, "clip_ratio/low_min": 3.255208442472698e-05, "clip_ratio/region_mean": 3.255208442472698e-05, "completions/clipped_ratio": 0.98333340883255, "completions/max_length": 2048.0, "completions/max_terminated_length": 418.0, "completions/mean_length": 2020.83349609375, "completions/mean_terminated_length": 418.0, "completions/min_length": 418.0, "completions/min_terminated_length": 418.0, "entropy": 11.473046461741129, "epoch": 0.4650145772594752, "frac_reward_zero_std": 0.0, "grad_norm": 0.03806354105472565, "learning_rate": 2e-05, "loss": 0.08692560344934464, "num_tokens": 22236275.0, "reward": -0.9867351055145264, "reward_std": 0.10275004804134369, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.9867350459098816, "rewards/length_penalty/std": 0.10275004059076309, "sampling/importance_sampling_ratio/max": 2.564708948135376, "sampling/importance_sampling_ratio/mean": 0.9487138986587524, "sampling/importance_sampling_ratio/min": 0.011752174235880375, "sampling/sampling_logp_difference/max": 4.443717002868652, "sampling/sampling_logp_difference/mean": 0.0713278129696846, "step": 319, "step_time": 32.62783778994344 }, { "clip_ratio/high_max": 5.772740648050482e-05, "clip_ratio/high_mean": 5.772740648050482e-05, "clip_ratio/low_mean": 0.00014863356166946082, "clip_ratio/low_min": 0.00014863356166946082, "clip_ratio/region_mean": 0.00020636096814996563, "completions/clipped_ratio": 0.9500000476837158, "completions/max_length": 2048.0, "completions/max_terminated_length": 391.0, "completions/mean_length": 1963.2333984375, "completions/mean_terminated_length": 352.66668701171875, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 11.266235828399658, "epoch": 0.46647230320699706, "frac_reward_zero_std": 0.0, "grad_norm": 0.05182773247361183, "learning_rate": 2e-05, "loss": 0.12751559913158417, "num_tokens": 22358629.0, "reward": -0.9252767562866211, "reward_std": 0.345950186252594, "rewards/correctness/mean": 0.03333333507180214, "rewards/correctness/std": 0.1810203194618225, "rewards/length_penalty/mean": -0.9586099982261658, "rewards/length_penalty/std": 0.18201181292533875, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9493284821510315, "sampling/importance_sampling_ratio/min": 0.09273446351289749, "sampling/sampling_logp_difference/max": 2.3780150413513184, "sampling/sampling_logp_difference/mean": 0.07015813142061234, "step": 320, "step_time": 31.40846612607129 }, { "clip_ratio/high_max": 2.5436065698158927e-05, "clip_ratio/high_mean": 2.5436065698158927e-05, "clip_ratio/low_mean": 0.00011879870483729367, "clip_ratio/low_min": 0.00011879870483729367, "clip_ratio/region_mean": 0.0001442347711417824, "completions/clipped_ratio": 0.9500000476837158, "completions/max_length": 2048.0, "completions/max_terminated_length": 838.0, "completions/mean_length": 1974.900146484375, "completions/mean_terminated_length": 586.0, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "entropy": 11.195334434509277, "epoch": 0.4679300291545189, "frac_reward_zero_std": 0.0, "grad_norm": 0.044528529047966, "learning_rate": 2e-05, "loss": 0.09148704260587692, "num_tokens": 22483053.0, "reward": -0.9143067002296448, "reward_std": 0.34867629408836365, "rewards/correctness/mean": 0.05000000074505806, "rewards/correctness/std": 0.21978417038917542, "rewards/length_penalty/mean": -0.964306652545929, "rewards/length_penalty/std": 0.16032728552818298, "sampling/importance_sampling_ratio/max": 2.9048712253570557, "sampling/importance_sampling_ratio/mean": 0.9496203064918518, "sampling/importance_sampling_ratio/min": 0.005555777810513973, "sampling/sampling_logp_difference/max": 5.1929168701171875, "sampling/sampling_logp_difference/mean": 0.06951345503330231, "step": 321, "step_time": 32.43216784996912 }, { "clip_ratio/high_max": 4.6104734792606905e-05, "clip_ratio/high_mean": 4.6104734792606905e-05, "clip_ratio/low_mean": 0.00010228719414347627, "clip_ratio/low_min": 0.00010228719414347627, "clip_ratio/region_mean": 0.00014839192651076397, "completions/clipped_ratio": 0.8666667342185974, "completions/max_length": 2048.0, "completions/max_terminated_length": 488.0, "completions/mean_length": 1818.1168212890625, "completions/mean_terminated_length": 323.875, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 11.211431662241617, "epoch": 0.46938775510204084, "frac_reward_zero_std": 0.0, "grad_norm": 0.02980216033756733, "learning_rate": 2e-05, "loss": 0.10076310485601425, "num_tokens": 22596050.0, "reward": -0.8710856437683105, "reward_std": 0.35448405146598816, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.8877522945404053, "rewards/length_penalty/std": 0.288849800825119, "sampling/importance_sampling_ratio/max": 2.4750120639801025, "sampling/importance_sampling_ratio/mean": 0.9501209855079651, "sampling/importance_sampling_ratio/min": 0.057793788611888885, "sampling/sampling_logp_difference/max": 2.8508739471435547, "sampling/sampling_logp_difference/mean": 0.06895281374454498, "step": 322, "step_time": 31.310074826003984 }, { "clip_ratio/high_max": 0.0002547116600908339, "clip_ratio/high_mean": 0.0002547116600908339, "clip_ratio/low_mean": 8.138020954599293e-06, "clip_ratio/low_min": 8.138020954599293e-06, "clip_ratio/region_mean": 0.0002628496804391034, "completions/clipped_ratio": 0.98333340883255, "completions/max_length": 2048.0, "completions/max_terminated_length": 192.0, "completions/mean_length": 2017.0667724609375, "completions/mean_terminated_length": 192.0, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 11.40187152226766, "epoch": 0.4708454810495627, "frac_reward_zero_std": 0.0, "grad_norm": 0.0341666117310524, "learning_rate": 2e-05, "loss": 0.10401123017072678, "num_tokens": 22720744.0, "reward": -0.9682292342185974, "reward_std": 0.24609582126140594, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9848958253860474, "rewards/length_penalty/std": 0.11699637025594711, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9496335983276367, "sampling/importance_sampling_ratio/min": 0.09663687646389008, "sampling/sampling_logp_difference/max": 2.336794853210449, "sampling/sampling_logp_difference/mean": 0.07031695544719696, "step": 323, "step_time": 31.43453913857229 }, { "clip_ratio/high_max": 0.0001998916753412535, "clip_ratio/high_mean": 0.0001998916753412535, "clip_ratio/low_mean": 4.882812633392556e-05, "clip_ratio/low_min": 4.882812633392556e-05, "clip_ratio/region_mean": 0.00024871980349416845, "completions/clipped_ratio": 0.9333333969116211, "completions/max_length": 2048.0, "completions/max_terminated_length": 449.0, "completions/mean_length": 1933.783447265625, "completions/mean_terminated_length": 334.75, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 11.316497643788656, "epoch": 0.47230320699708456, "frac_reward_zero_std": 0.0, "grad_norm": 0.031590815633535385, "learning_rate": 2e-05, "loss": 0.0900404155254364, "num_tokens": 22841301.0, "reward": -0.9275635480880737, "reward_std": 0.29260629415512085, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9442301392555237, "rewards/length_penalty/std": 0.21065562963485718, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9498494267463684, "sampling/importance_sampling_ratio/min": 0.04626120626926422, "sampling/sampling_logp_difference/max": 3.073451519012451, "sampling/sampling_logp_difference/mean": 0.0701475441455841, "step": 324, "step_time": 31.86355043691583 }, { "clip_ratio/high_max": 0.00034319468068133574, "clip_ratio/high_mean": 0.00034319468068133574, "clip_ratio/low_mean": 2.784323138863935e-05, "clip_ratio/low_min": 2.784323138863935e-05, "clip_ratio/region_mean": 0.0003710379144952943, "completions/clipped_ratio": 0.8833333849906921, "completions/max_length": 2048.0, "completions/max_terminated_length": 370.0, "completions/mean_length": 1844.9334716796875, "completions/mean_terminated_length": 307.4285888671875, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 11.309606234232584, "epoch": 0.4737609329446064, "frac_reward_zero_std": 0.0, "grad_norm": 0.050031911581754684, "learning_rate": 2e-05, "loss": 0.21805138885974884, "num_tokens": 22956077.0, "reward": -0.7841797471046448, "reward_std": 0.5989142060279846, "rewards/correctness/mean": 0.11666666716337204, "rewards/correctness/std": 0.32373178005218506, "rewards/length_penalty/mean": -0.9008463621139526, "rewards/length_penalty/std": 0.2752372622489929, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9496719837188721, "sampling/importance_sampling_ratio/min": 0.01406992506235838, "sampling/sampling_logp_difference/max": 4.263715744018555, "sampling/sampling_logp_difference/mean": 0.07009508460760117, "step": 325, "step_time": 30.925921589136124 }, { "clip_ratio/high_max": 0.00013618464132984323, "clip_ratio/high_mean": 0.00013618464132984323, "clip_ratio/low_mean": 7.407858962930429e-05, "clip_ratio/low_min": 7.407858962930429e-05, "clip_ratio/region_mean": 0.00021026323823510515, "completions/clipped_ratio": 0.8833333849906921, "completions/max_length": 2048.0, "completions/max_terminated_length": 865.0, "completions/mean_length": 1859.666748046875, "completions/mean_terminated_length": 433.71429443359375, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 11.094075838724772, "epoch": 0.4752186588921283, "frac_reward_zero_std": 0.0, "grad_norm": 0.02850618213415146, "learning_rate": 2e-05, "loss": 0.07809044420719147, "num_tokens": 23072557.0, "reward": -0.8913737535476685, "reward_std": 0.32225164771080017, "rewards/correctness/mean": 0.01666666753590107, "rewards/correctness/std": 0.12909944355487823, "rewards/length_penalty/mean": -0.9080403447151184, "rewards/length_penalty/std": 0.25924280285835266, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9511896371841431, "sampling/importance_sampling_ratio/min": 0.008387433364987373, "sampling/sampling_logp_difference/max": 4.781020641326904, "sampling/sampling_logp_difference/mean": 0.06822741776704788, "step": 326, "step_time": 31.41011229204014 }, { "clip_ratio/high_max": 3.656538562305892e-05, "clip_ratio/high_mean": 3.656538562305892e-05, "clip_ratio/low_mean": 5.621760161981607e-05, "clip_ratio/low_min": 5.621760161981607e-05, "clip_ratio/region_mean": 9.27829866365452e-05, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 329.0, "completions/mean_length": 1780.10009765625, "completions/mean_terminated_length": 262.0, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 11.27370309829712, "epoch": 0.47667638483965014, "frac_reward_zero_std": 0.0, "grad_norm": 0.05238368734717369, "learning_rate": 2e-05, "loss": 0.20817898213863373, "num_tokens": 23183353.0, "reward": -0.719189465045929, "reward_std": 0.651321530342102, "rewards/correctness/mean": 0.15000000596046448, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.869189441204071, "rewards/length_penalty/std": 0.3141154646873474, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9501121044158936, "sampling/importance_sampling_ratio/min": 0.04895056411623955, "sampling/sampling_logp_difference/max": 3.016944408416748, "sampling/sampling_logp_difference/mean": 0.06942541152238846, "step": 327, "step_time": 30.601299125235528 }, { "clip_ratio/high_max": 3.682100092798161e-05, "clip_ratio/high_mean": 3.682100092798161e-05, "clip_ratio/low_mean": 2.513657697515252e-05, "clip_ratio/low_min": 2.513657697515252e-05, "clip_ratio/region_mean": 6.195757790313412e-05, "completions/clipped_ratio": 0.8833333849906921, "completions/max_length": 2048.0, "completions/max_terminated_length": 795.0, "completions/mean_length": 1874.0833740234375, "completions/mean_terminated_length": 557.2857666015625, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 10.884379386901855, "epoch": 0.478134110787172, "frac_reward_zero_std": 0.0, "grad_norm": 0.03748556971549988, "learning_rate": 2e-05, "loss": 0.12448544055223465, "num_tokens": 23299758.0, "reward": -0.8650798201560974, "reward_std": 0.4074273109436035, "rewards/correctness/mean": 0.05000000074505806, "rewards/correctness/std": 0.21978417038917542, "rewards/length_penalty/mean": -0.9150797724723816, "rewards/length_penalty/std": 0.23737405240535736, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.951575756072998, "sampling/importance_sampling_ratio/min": 0.26177921891212463, "sampling/sampling_logp_difference/max": 2.4633121490478516, "sampling/sampling_logp_difference/mean": 0.0681769847869873, "step": 328, "step_time": 31.43399061821401 }, { "clip_ratio/high_max": 5.603051310269317e-05, "clip_ratio/high_mean": 5.603051310269317e-05, "clip_ratio/low_mean": 0.00011998377946535281, "clip_ratio/low_min": 0.00011998377946535281, "clip_ratio/region_mean": 0.0001760142968123546, "completions/clipped_ratio": 0.8333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 503.0, "completions/mean_length": 1756.2000732421875, "completions/mean_terminated_length": 297.20001220703125, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 11.016518592834473, "epoch": 0.47959183673469385, "frac_reward_zero_std": 0.0, "grad_norm": 0.04502980411052704, "learning_rate": 2e-05, "loss": 0.19054926931858063, "num_tokens": 23409950.0, "reward": -0.757519543170929, "reward_std": 0.5865859985351562, "rewards/correctness/mean": 0.10000000149011612, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.8575195074081421, "rewards/length_penalty/std": 0.32179832458496094, "sampling/importance_sampling_ratio/max": 2.9724059104919434, "sampling/importance_sampling_ratio/mean": 0.950715959072113, "sampling/importance_sampling_ratio/min": 0.11847664415836334, "sampling/sampling_logp_difference/max": 2.1330394744873047, "sampling/sampling_logp_difference/mean": 0.06767094880342484, "step": 329, "step_time": 31.301492590922862 }, { "clip_ratio/high_max": 0.00013489045886672102, "clip_ratio/high_mean": 0.00013489045886672102, "clip_ratio/low_mean": 0.00018161665138904937, "clip_ratio/low_min": 0.00018161665138904937, "clip_ratio/region_mean": 0.00031650710297981277, "completions/clipped_ratio": 0.8333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 464.0, "completions/mean_length": 1757.4833984375, "completions/mean_terminated_length": 304.8999938964844, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 10.910406907399496, "epoch": 0.48104956268221577, "frac_reward_zero_std": 0.0, "grad_norm": 0.044379498809576035, "learning_rate": 2e-05, "loss": 0.21004167199134827, "num_tokens": 23519919.0, "reward": -0.7414795160293579, "reward_std": 0.5960094332695007, "rewards/correctness/mean": 0.11666666716337204, "rewards/correctness/std": 0.32373178005218506, "rewards/length_penalty/mean": -0.8581461310386658, "rewards/length_penalty/std": 0.3206380605697632, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.951592206954956, "sampling/importance_sampling_ratio/min": 0.033146366477012634, "sampling/sampling_logp_difference/max": 3.4068222045898438, "sampling/sampling_logp_difference/mean": 0.0668276995420456, "step": 330, "step_time": 30.477692388929427 }, { "clip_ratio/high_max": 0.00043264226405881345, "clip_ratio/high_mean": 0.00043264226405881345, "clip_ratio/low_mean": 2.0789647048028808e-05, "clip_ratio/low_min": 2.0789647048028808e-05, "clip_ratio/region_mean": 0.00045343191595748067, "completions/clipped_ratio": 0.7833333611488342, "completions/max_length": 2048.0, "completions/max_terminated_length": 435.0, "completions/mean_length": 1662.36669921875, "completions/mean_terminated_length": 268.15386962890625, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 11.00306765238444, "epoch": 0.48250728862973763, "frac_reward_zero_std": 0.0, "grad_norm": 0.04891810193657875, "learning_rate": 2e-05, "loss": 0.2580132484436035, "num_tokens": 23624461.0, "reward": -0.6117025017738342, "reward_std": 0.7561345100402832, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.8117024898529053, "rewards/length_penalty/std": 0.36154839396476746, "sampling/importance_sampling_ratio/max": 2.4825069904327393, "sampling/importance_sampling_ratio/mean": 0.9511086940765381, "sampling/importance_sampling_ratio/min": 0.026767052710056305, "sampling/sampling_logp_difference/max": 3.6205835342407227, "sampling/sampling_logp_difference/mean": 0.06597679108381271, "step": 331, "step_time": 31.260890210513026 }, { "clip_ratio/high_max": 3.975756105016141e-05, "clip_ratio/high_mean": 3.975756105016141e-05, "clip_ratio/low_mean": 0.00012677132205377953, "clip_ratio/low_min": 0.00012677132205377953, "clip_ratio/region_mean": 0.00016652888128495155, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 606.0, "completions/mean_length": 1802.466796875, "completions/mean_terminated_length": 411.1111145019531, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 11.079758008321127, "epoch": 0.4839650145772595, "frac_reward_zero_std": 0.0, "grad_norm": 0.04108228534460068, "learning_rate": 2e-05, "loss": 0.1960233449935913, "num_tokens": 23737849.0, "reward": -0.746777355670929, "reward_std": 0.6010469794273376, "rewards/correctness/mean": 0.13333334028720856, "rewards/correctness/std": 0.34280332922935486, "rewards/length_penalty/mean": -0.8801106810569763, "rewards/length_penalty/std": 0.28886422514915466, "sampling/importance_sampling_ratio/max": 2.530056953430176, "sampling/importance_sampling_ratio/mean": 0.9506701827049255, "sampling/importance_sampling_ratio/min": 0.012197640724480152, "sampling/sampling_logp_difference/max": 4.40651273727417, "sampling/sampling_logp_difference/mean": 0.06576238572597504, "step": 332, "step_time": 30.88202503300272 }, { "clip_ratio/high_max": 0.00012358402091194876, "clip_ratio/high_mean": 0.00012358402091194876, "clip_ratio/low_mean": 0.00014749099379211353, "clip_ratio/low_min": 0.00014749099379211353, "clip_ratio/region_mean": 0.0002710750147040623, "completions/clipped_ratio": 0.7333333492279053, "completions/max_length": 2048.0, "completions/max_terminated_length": 1515.0, "completions/mean_length": 1609.0667724609375, "completions/mean_terminated_length": 402.0, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 10.905617872873941, "epoch": 0.48542274052478135, "frac_reward_zero_std": 0.0, "grad_norm": 0.0504484660923481, "learning_rate": 2e-05, "loss": 0.23672306537628174, "num_tokens": 23837833.0, "reward": -0.6356770992279053, "reward_std": 0.6742239594459534, "rewards/correctness/mean": 0.15000000596046448, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.7856770753860474, "rewards/length_penalty/std": 0.366056352853775, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9515051245689392, "sampling/importance_sampling_ratio/min": 0.017611064016819, "sampling/sampling_logp_difference/max": 4.0392279624938965, "sampling/sampling_logp_difference/mean": 0.06583934277296066, "step": 333, "step_time": 30.436302243266255 }, { "clip_ratio/high_max": 4.1316900023957714e-05, "clip_ratio/high_mean": 4.1316900023957714e-05, "clip_ratio/low_mean": 0.00011513096675723015, "clip_ratio/low_min": 0.00011513096675723015, "clip_ratio/region_mean": 0.00015644786556852827, "completions/clipped_ratio": 0.8333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 424.0, "completions/mean_length": 1760.4833984375, "completions/mean_terminated_length": 322.8999938964844, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 11.157856941223145, "epoch": 0.4868804664723032, "frac_reward_zero_std": 0.0, "grad_norm": 0.0515957809984684, "learning_rate": 2e-05, "loss": 0.227167546749115, "num_tokens": 23948552.0, "reward": -0.692944347858429, "reward_std": 0.6925169229507446, "rewards/correctness/mean": 0.1666666716337204, "rewards/correctness/std": 0.3758230209350586, "rewards/length_penalty/mean": -0.8596109747886658, "rewards/length_penalty/std": 0.3168426752090454, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9503925442695618, "sampling/importance_sampling_ratio/min": 0.0527186393737793, "sampling/sampling_logp_difference/max": 2.94278621673584, "sampling/sampling_logp_difference/mean": 0.06562060117721558, "step": 334, "step_time": 30.5838089894969 }, { "clip_ratio/high_max": 5.1860941312042996e-05, "clip_ratio/high_mean": 5.1860941312042996e-05, "clip_ratio/low_mean": 0.00013514182986303544, "clip_ratio/low_min": 0.00013514182986303544, "clip_ratio/region_mean": 0.00018700277178140823, "completions/clipped_ratio": 0.9333333969116211, "completions/max_length": 2048.0, "completions/max_terminated_length": 467.0, "completions/mean_length": 1936.8333740234375, "completions/mean_terminated_length": 380.5, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 11.305522600809732, "epoch": 0.48833819241982507, "frac_reward_zero_std": 0.0, "grad_norm": 0.040108662098646164, "learning_rate": 2e-05, "loss": 0.114908866584301, "num_tokens": 24071742.0, "reward": -0.8790527582168579, "reward_std": 0.4221198558807373, "rewards/correctness/mean": 0.06666667014360428, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.9457194209098816, "rewards/length_penalty/std": 0.20516087114810944, "sampling/importance_sampling_ratio/max": 2.7430994510650635, "sampling/importance_sampling_ratio/mean": 0.9497591853141785, "sampling/importance_sampling_ratio/min": 0.007772922515869141, "sampling/sampling_logp_difference/max": 4.857109069824219, "sampling/sampling_logp_difference/mean": 0.06709293276071548, "step": 335, "step_time": 33.456730286823586 }, { "clip_ratio/high_max": 0.00016312114530592225, "clip_ratio/high_mean": 0.00016312114530592225, "clip_ratio/low_mean": 5.428145777841564e-05, "clip_ratio/low_min": 5.428145777841564e-05, "clip_ratio/region_mean": 0.0002174026049033273, "completions/clipped_ratio": 0.8500000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 676.0, "completions/mean_length": 1822.5833740234375, "completions/mean_terminated_length": 545.2222290039062, "completions/min_length": 459.0, "completions/min_terminated_length": 459.0, "entropy": 10.606079896291098, "epoch": 0.4897959183673469, "frac_reward_zero_std": 0.0, "grad_norm": 0.0445568785071373, "learning_rate": 2e-05, "loss": 0.04573051258921623, "num_tokens": 24187117.0, "reward": -0.8399333357810974, "reward_std": 0.36278223991394043, "rewards/correctness/mean": 0.05000000074505806, "rewards/correctness/std": 0.21978417038917542, "rewards/length_penalty/mean": -0.8899332880973816, "rewards/length_penalty/std": 0.2645168900489807, "sampling/importance_sampling_ratio/max": 2.298827886581421, "sampling/importance_sampling_ratio/mean": 0.9529029130935669, "sampling/importance_sampling_ratio/min": 0.10117930173873901, "sampling/sampling_logp_difference/max": 2.290861129760742, "sampling/sampling_logp_difference/mean": 0.06351746618747711, "step": 336, "step_time": 31.45081715611741 }, { "clip_ratio/high_max": 0.00010574442300518665, "clip_ratio/high_mean": 0.00010574442300518665, "clip_ratio/low_mean": 3.471884034903875e-05, "clip_ratio/low_min": 3.471884034903875e-05, "clip_ratio/region_mean": 0.0001404632639605552, "completions/clipped_ratio": 0.8000000715255737, "completions/max_length": 2048.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 1720.283447265625, "completions/mean_terminated_length": 409.41668701171875, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 10.821594874064127, "epoch": 0.4912536443148688, "frac_reward_zero_std": 0.0, "grad_norm": 0.05119682848453522, "learning_rate": 2e-05, "loss": 0.2311152219772339, "num_tokens": 24296624.0, "reward": -0.6566488146781921, "reward_std": 0.6844743490219116, "rewards/correctness/mean": 0.18333333730697632, "rewards/correctness/std": 0.39020493626594543, "rewards/length_penalty/mean": -0.8399820923805237, "rewards/length_penalty/std": 0.324534147977829, "sampling/importance_sampling_ratio/max": 2.1988935470581055, "sampling/importance_sampling_ratio/mean": 0.951712965965271, "sampling/importance_sampling_ratio/min": 0.04199759662151337, "sampling/sampling_logp_difference/max": 3.170142889022827, "sampling/sampling_logp_difference/mean": 0.06512091308832169, "step": 337, "step_time": 31.427040182054043 }, { "clip_ratio/high_max": 0.00015063554807663118, "clip_ratio/high_mean": 0.00015063554807663118, "clip_ratio/low_mean": 6.29767740368455e-05, "clip_ratio/low_min": 6.29767740368455e-05, "clip_ratio/region_mean": 0.00021361231544384887, "completions/clipped_ratio": 0.8000000715255737, "completions/max_length": 2048.0, "completions/max_terminated_length": 787.0, "completions/mean_length": 1730.650146484375, "completions/mean_terminated_length": 461.25, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 10.729857921600342, "epoch": 0.49271137026239065, "frac_reward_zero_std": 0.0, "grad_norm": 0.04778337851166725, "learning_rate": 2e-05, "loss": 0.21082454919815063, "num_tokens": 24406293.0, "reward": -0.7450439929962158, "reward_std": 0.575949490070343, "rewards/correctness/mean": 0.10000000149011612, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.845043957233429, "rewards/length_penalty/std": 0.31483665108680725, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.952362596988678, "sampling/importance_sampling_ratio/min": 0.02548110857605934, "sampling/sampling_logp_difference/max": 3.6698179244995117, "sampling/sampling_logp_difference/mean": 0.06484726816415787, "step": 338, "step_time": 30.73781864857301 }, { "clip_ratio/high_max": 0.000222121451467198, "clip_ratio/high_mean": 0.000222121451467198, "clip_ratio/low_mean": 6.255518140581746e-05, "clip_ratio/low_min": 6.255518140581746e-05, "clip_ratio/region_mean": 0.00028467663044769626, "completions/clipped_ratio": 0.7166666984558105, "completions/max_length": 2048.0, "completions/max_terminated_length": 1311.0, "completions/mean_length": 1585.966796875, "completions/mean_terminated_length": 417.29412841796875, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 10.743321418762207, "epoch": 0.49416909620991256, "frac_reward_zero_std": 0.0, "grad_norm": 0.03772800415754318, "learning_rate": 2e-05, "loss": 0.19322876632213593, "num_tokens": 24506921.0, "reward": -0.6243978142738342, "reward_std": 0.6787324547767639, "rewards/correctness/mean": 0.15000000596046448, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.7743977904319763, "rewards/length_penalty/std": 0.3706940710544586, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9521399736404419, "sampling/importance_sampling_ratio/min": 0.05523095652461052, "sampling/sampling_logp_difference/max": 2.8962316513061523, "sampling/sampling_logp_difference/mean": 0.06458830833435059, "step": 339, "step_time": 30.833504950162023 }, { "clip_ratio/high_max": 0.00012167230791722734, "clip_ratio/high_mean": 0.00012167230791722734, "clip_ratio/low_mean": 0.00012218104954323886, "clip_ratio/low_min": 0.00012218104954323886, "clip_ratio/region_mean": 0.0002438533556414768, "completions/clipped_ratio": 0.7333333492279053, "completions/max_length": 2048.0, "completions/max_terminated_length": 1732.0, "completions/mean_length": 1632.7501220703125, "completions/mean_terminated_length": 490.8125, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "entropy": 10.686214447021484, "epoch": 0.4956268221574344, "frac_reward_zero_std": 0.0, "grad_norm": 0.046971991658210754, "learning_rate": 2e-05, "loss": 0.17797572910785675, "num_tokens": 24611636.0, "reward": -0.7139078974723816, "reward_std": 0.5638810396194458, "rewards/correctness/mean": 0.0833333358168602, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.7972412109375, "rewards/length_penalty/std": 0.3585253357887268, "sampling/importance_sampling_ratio/max": 2.098437547683716, "sampling/importance_sampling_ratio/mean": 0.9526610374450684, "sampling/importance_sampling_ratio/min": 0.025923501700162888, "sampling/sampling_logp_difference/max": 3.6526052951812744, "sampling/sampling_logp_difference/mean": 0.0641532838344574, "step": 340, "step_time": 30.814440657617524 }, { "clip_ratio/high_max": 0.00011522928980411962, "clip_ratio/high_mean": 0.00011522928980411962, "clip_ratio/low_mean": 8.31723179241332e-05, "clip_ratio/low_min": 8.31723179241332e-05, "clip_ratio/region_mean": 0.00019840160651559321, "completions/clipped_ratio": 0.7333333492279053, "completions/max_length": 2048.0, "completions/max_terminated_length": 2016.0, "completions/mean_length": 1610.10009765625, "completions/mean_terminated_length": 405.875, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 10.76656182607015, "epoch": 0.4970845481049563, "frac_reward_zero_std": 0.0, "grad_norm": 0.049459561705589294, "learning_rate": 2e-05, "loss": 0.07491716742515564, "num_tokens": 24712612.0, "reward": -0.6861816644668579, "reward_std": 0.5510381460189819, "rewards/correctness/mean": 0.10000000149011612, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.786181628704071, "rewards/length_penalty/std": 0.3734009265899658, "sampling/importance_sampling_ratio/max": 1.9498199224472046, "sampling/importance_sampling_ratio/mean": 0.9525728225708008, "sampling/importance_sampling_ratio/min": 0.15589269995689392, "sampling/sampling_logp_difference/max": 1.858587384223938, "sampling/sampling_logp_difference/mean": 0.06421062350273132, "step": 341, "step_time": 30.692465445725247 }, { "clip_ratio/high_max": 0.0002962095823022537, "clip_ratio/high_mean": 0.0002962095823022537, "clip_ratio/low_mean": 6.428931252836871e-05, "clip_ratio/low_min": 6.428931252836871e-05, "clip_ratio/region_mean": 0.0003604988984686012, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 433.0, "completions/mean_length": 1339.916748046875, "completions/mean_terminated_length": 277.79168701171875, "completions/min_length": 164.0, "completions/min_terminated_length": 164.0, "entropy": 10.181580066680908, "epoch": 0.49854227405247814, "frac_reward_zero_std": 0.0, "grad_norm": 0.05168772488832474, "learning_rate": 2e-05, "loss": 0.19935443997383118, "num_tokens": 24797267.0, "reward": -0.27092286944389343, "reward_std": 0.8936313986778259, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.6542561650276184, "rewards/length_penalty/std": 0.42755061388015747, "sampling/importance_sampling_ratio/max": 2.7307679653167725, "sampling/importance_sampling_ratio/mean": 0.9546541571617126, "sampling/importance_sampling_ratio/min": 0.07636299729347229, "sampling/sampling_logp_difference/max": 2.5722570419311523, "sampling/sampling_logp_difference/mean": 0.06101895868778229, "step": 342, "step_time": 28.787516271928325 }, { "clip_ratio/high_max": 0.00016324288177808435, "clip_ratio/high_mean": 0.00016324288177808435, "clip_ratio/low_mean": 0.00013309042697073892, "clip_ratio/low_min": 0.00013309042697073892, "clip_ratio/region_mean": 0.0002963333099614829, "completions/clipped_ratio": 0.6333333849906921, "completions/max_length": 2048.0, "completions/max_terminated_length": 1047.0, "completions/mean_length": 1438.4500732421875, "completions/mean_terminated_length": 385.5909118652344, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 10.255228042602539, "epoch": 0.5, "frac_reward_zero_std": 0.0, "grad_norm": 0.04742039367556572, "learning_rate": 2e-05, "loss": 0.15911054611206055, "num_tokens": 24889754.0, "reward": -0.5523681640625, "reward_std": 0.6759516596794128, "rewards/correctness/mean": 0.15000000596046448, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.7023681402206421, "rewards/length_penalty/std": 0.39749911427497864, "sampling/importance_sampling_ratio/max": 2.464223623275757, "sampling/importance_sampling_ratio/mean": 0.9537211656570435, "sampling/importance_sampling_ratio/min": 0.1596931368112564, "sampling/sampling_logp_difference/max": 1.8345012664794922, "sampling/sampling_logp_difference/mean": 0.06152910739183426, "step": 343, "step_time": 30.191806897055358 }, { "clip_ratio/high_max": 0.0001864053774625063, "clip_ratio/high_mean": 0.0001864053774625063, "clip_ratio/low_mean": 9.824021920697608e-05, "clip_ratio/low_min": 9.824021920697608e-05, "clip_ratio/region_mean": 0.000284645600913791, "completions/clipped_ratio": 0.6500000357627869, "completions/max_length": 2048.0, "completions/max_terminated_length": 796.0, "completions/mean_length": 1470.10009765625, "completions/mean_terminated_length": 396.8571472167969, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 10.249218304951986, "epoch": 0.5014577259475219, "frac_reward_zero_std": 0.0, "grad_norm": 0.0327146016061306, "learning_rate": 2e-05, "loss": 0.11380994319915771, "num_tokens": 24983050.0, "reward": -0.6678223013877869, "reward_std": 0.49976953864097595, "rewards/correctness/mean": 0.05000000074505806, "rewards/correctness/std": 0.2197841852903366, "rewards/length_penalty/mean": -0.717822253704071, "rewards/length_penalty/std": 0.39048999547958374, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9540876746177673, "sampling/importance_sampling_ratio/min": 0.02695874124765396, "sampling/sampling_logp_difference/max": 3.613447666168213, "sampling/sampling_logp_difference/mean": 0.060118578374385834, "step": 344, "step_time": 29.731167372781783 }, { "clip_ratio/high_max": 0.00016021922298629457, "clip_ratio/high_mean": 0.00016021922298629457, "clip_ratio/low_mean": 0.00022324789824779145, "clip_ratio/low_min": 0.00022324789824779145, "clip_ratio/region_mean": 0.0003834671200214264, "completions/clipped_ratio": 0.7000000476837158, "completions/max_length": 2048.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 1556.783447265625, "completions/mean_terminated_length": 410.6111145019531, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 9.993067741394043, "epoch": 0.5029154518950437, "frac_reward_zero_std": 0.0, "grad_norm": 0.037616100162267685, "learning_rate": 2e-05, "loss": 0.19770270586013794, "num_tokens": 25081137.0, "reward": -0.5601481199264526, "reward_std": 0.7313750386238098, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.7601481080055237, "rewards/length_penalty/std": 0.3727264404296875, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9549320936203003, "sampling/importance_sampling_ratio/min": 0.012503791600465775, "sampling/sampling_logp_difference/max": 4.381723403930664, "sampling/sampling_logp_difference/mean": 0.05891606584191322, "step": 345, "step_time": 29.80520933587104 }, { "clip_ratio/high_max": 0.00017083743781161806, "clip_ratio/high_mean": 0.00017083743781161806, "clip_ratio/low_mean": 0.00012933207835885696, "clip_ratio/low_min": 0.00012933207835885696, "clip_ratio/region_mean": 0.000300169516170475, "completions/clipped_ratio": 0.7333333492279053, "completions/max_length": 2048.0, "completions/max_terminated_length": 594.0, "completions/mean_length": 1603.6334228515625, "completions/mean_terminated_length": 381.625, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 10.545932133992514, "epoch": 0.5043731778425656, "frac_reward_zero_std": 0.0, "grad_norm": 0.04711635783314705, "learning_rate": 2e-05, "loss": 0.14977839589118958, "num_tokens": 25183375.0, "reward": -0.6830241084098816, "reward_std": 0.5773727297782898, "rewards/correctness/mean": 0.10000000149011612, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.7830240726470947, "rewards/length_penalty/std": 0.3644307255744934, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9528772830963135, "sampling/importance_sampling_ratio/min": 0.0020915709901601076, "sampling/sampling_logp_difference/max": 6.169839859008789, "sampling/sampling_logp_difference/mean": 0.05956917628645897, "step": 346, "step_time": 30.992498975014314 }, { "clip_ratio/high_max": 0.0005137092375662178, "clip_ratio/high_mean": 0.0005137092375662178, "clip_ratio/low_mean": 9.593774908959556e-05, "clip_ratio/low_min": 9.593774908959556e-05, "clip_ratio/region_mean": 0.0006096469878684729, "completions/clipped_ratio": 0.38333335518836975, "completions/max_length": 2048.0, "completions/max_terminated_length": 525.0, "completions/mean_length": 1008.4667358398438, "completions/mean_terminated_length": 362.2702941894531, "completions/min_length": 159.0, "completions/min_terminated_length": 159.0, "entropy": 9.065423806508383, "epoch": 0.5058309037900874, "frac_reward_zero_std": 0.0, "grad_norm": 0.05288033187389374, "learning_rate": 2e-05, "loss": 0.30863136053085327, "num_tokens": 25246793.0, "reward": -0.07574870437383652, "reward_std": 0.8338004946708679, "rewards/correctness/mean": 0.4166666567325592, "rewards/correctness/std": 0.49716717004776, "rewards/length_penalty/mean": -0.4924153685569763, "rewards/length_penalty/std": 0.4053652286529541, "sampling/importance_sampling_ratio/max": 2.5560455322265625, "sampling/importance_sampling_ratio/mean": 0.9585880041122437, "sampling/importance_sampling_ratio/min": 0.0007648623432032764, "sampling/sampling_logp_difference/max": 7.175814628601074, "sampling/sampling_logp_difference/mean": 0.05526462197303772, "step": 347, "step_time": 26.743368809577078 }, { "clip_ratio/high_max": 0.00014060297204802433, "clip_ratio/high_mean": 0.00014060297204802433, "clip_ratio/low_mean": 9.701850528169113e-05, "clip_ratio/low_min": 9.701850528169113e-05, "clip_ratio/region_mean": 0.00023762147854237506, "completions/clipped_ratio": 0.5833333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 1343.683349609375, "completions/mean_terminated_length": 357.6399841308594, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 10.32764466603597, "epoch": 0.5072886297376094, "frac_reward_zero_std": 0.0, "grad_norm": 0.054406147450208664, "learning_rate": 2e-05, "loss": 0.1929289549589157, "num_tokens": 25331734.0, "reward": -0.40609538555145264, "reward_std": 0.7531402707099915, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.6560953855514526, "rewards/length_penalty/std": 0.4123750329017639, "sampling/importance_sampling_ratio/max": 2.553048849105835, "sampling/importance_sampling_ratio/mean": 0.9542434811592102, "sampling/importance_sampling_ratio/min": 0.004802689887583256, "sampling/sampling_logp_difference/max": 5.338579177856445, "sampling/sampling_logp_difference/mean": 0.05878261476755142, "step": 348, "step_time": 29.425849793246016 }, { "clip_ratio/high_max": 0.0001381195082406824, "clip_ratio/high_mean": 0.0001381195082406824, "clip_ratio/low_mean": 9.491529817751143e-05, "clip_ratio/low_min": 9.491529817751143e-05, "clip_ratio/region_mean": 0.00023303480702452362, "completions/clipped_ratio": 0.6333333849906921, "completions/max_length": 2048.0, "completions/max_terminated_length": 1583.0, "completions/mean_length": 1463.0833740234375, "completions/mean_terminated_length": 452.7727355957031, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 10.205944061279297, "epoch": 0.5087463556851312, "frac_reward_zero_std": 0.0, "grad_norm": 0.03182622417807579, "learning_rate": 2e-05, "loss": 0.1862814575433731, "num_tokens": 25425749.0, "reward": -0.5643962025642395, "reward_std": 0.6425461173057556, "rewards/correctness/mean": 0.15000000596046448, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.7143961787223816, "rewards/length_penalty/std": 0.3868906497955322, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9543356895446777, "sampling/importance_sampling_ratio/min": 0.14406640827655792, "sampling/sampling_logp_difference/max": 1.9374809265136719, "sampling/sampling_logp_difference/mean": 0.06057218089699745, "step": 349, "step_time": 29.682311285985634 }, { "clip_ratio/high_max": 0.00011108685854802995, "clip_ratio/high_mean": 0.00011108685854802995, "clip_ratio/low_mean": 0.00016860874651077515, "clip_ratio/low_min": 0.00016860874651077515, "clip_ratio/region_mean": 0.0002796956032398157, "completions/clipped_ratio": 0.5666667222976685, "completions/max_length": 2048.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 1306.10009765625, "completions/mean_terminated_length": 335.923095703125, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 10.19064466158549, "epoch": 0.5102040816326531, "frac_reward_zero_std": 0.0, "grad_norm": 0.0432235486805439, "learning_rate": 2e-05, "loss": 0.20542196929454803, "num_tokens": 25509215.0, "reward": -0.38774415850639343, "reward_std": 0.7839159965515137, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.637744128704071, "rewards/length_penalty/std": 0.41901785135269165, "sampling/importance_sampling_ratio/max": 2.522533893585205, "sampling/importance_sampling_ratio/mean": 0.9529611468315125, "sampling/importance_sampling_ratio/min": 0.05310524255037308, "sampling/sampling_logp_difference/max": 2.9354796409606934, "sampling/sampling_logp_difference/mean": 0.06207723170518875, "step": 350, "step_time": 28.715401058085263 }, { "clip_ratio/high_max": 0.00030635641026795685, "clip_ratio/high_mean": 0.00030635641026795685, "clip_ratio/low_mean": 0.0001011270796880126, "clip_ratio/low_min": 0.0001011270796880126, "clip_ratio/region_mean": 0.00040748349177495885, "completions/clipped_ratio": 0.6333333849906921, "completions/max_length": 2048.0, "completions/max_terminated_length": 626.0, "completions/mean_length": 1397.166748046875, "completions/mean_terminated_length": 273.0, "completions/min_length": 121.0, "completions/min_terminated_length": 121.0, "entropy": 10.380496819814047, "epoch": 0.5116618075801749, "frac_reward_zero_std": 0.0, "grad_norm": 0.04086378216743469, "learning_rate": 2e-05, "loss": 0.18143241107463837, "num_tokens": 25597175.0, "reward": -0.5322102904319763, "reward_std": 0.6913964152336121, "rewards/correctness/mean": 0.15000000596046448, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.6822102665901184, "rewards/length_penalty/std": 0.42241907119750977, "sampling/importance_sampling_ratio/max": 2.0262372493743896, "sampling/importance_sampling_ratio/mean": 0.9534326195716858, "sampling/importance_sampling_ratio/min": 0.015885965898633003, "sampling/sampling_logp_difference/max": 4.142319202423096, "sampling/sampling_logp_difference/mean": 0.06144274026155472, "step": 351, "step_time": 29.458701271330938 }, { "clip_ratio/high_max": 0.00022556335655584311, "clip_ratio/high_mean": 0.00022556335655584311, "clip_ratio/low_mean": 6.820335026228956e-05, "clip_ratio/low_min": 6.820335026228956e-05, "clip_ratio/region_mean": 0.0002937667092434519, "completions/clipped_ratio": 0.7166666984558105, "completions/max_length": 2048.0, "completions/max_terminated_length": 1917.0, "completions/mean_length": 1614.2333984375, "completions/mean_terminated_length": 517.058837890625, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 10.294501940409342, "epoch": 0.5131195335276968, "frac_reward_zero_std": 0.0, "grad_norm": 0.0491446815431118, "learning_rate": 2e-05, "loss": 0.23559007048606873, "num_tokens": 25703569.0, "reward": -0.5715332627296448, "reward_std": 0.7282727956771851, "rewards/correctness/mean": 0.21666666865348816, "rewards/correctness/std": 0.41545018553733826, "rewards/length_penalty/mean": -0.7881998419761658, "rewards/length_penalty/std": 0.3556015193462372, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9541250467300415, "sampling/importance_sampling_ratio/min": 0.04362950101494789, "sampling/sampling_logp_difference/max": 3.13202166557312, "sampling/sampling_logp_difference/mean": 0.06071099266409874, "step": 352, "step_time": 31.697401638142765 }, { "clip_ratio/high_max": 0.0001684131214763814, "clip_ratio/high_mean": 0.0001684131214763814, "clip_ratio/low_mean": 0.00014855047872212404, "clip_ratio/low_min": 0.00014855047872212404, "clip_ratio/region_mean": 0.00031696359898584586, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 897.0, "completions/mean_length": 1429.050048828125, "completions/mean_terminated_length": 500.625, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 9.636358578999838, "epoch": 0.5145772594752187, "frac_reward_zero_std": 0.0, "grad_norm": 0.037650760263204575, "learning_rate": 2e-05, "loss": 0.20597147941589355, "num_tokens": 25793712.0, "reward": -0.4477783441543579, "reward_std": 0.7487742304801941, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.6977783441543579, "rewards/length_penalty/std": 0.3801999092102051, "sampling/importance_sampling_ratio/max": 2.7380683422088623, "sampling/importance_sampling_ratio/mean": 0.9566221833229065, "sampling/importance_sampling_ratio/min": 0.00025494524743407965, "sampling/sampling_logp_difference/max": 8.27446174621582, "sampling/sampling_logp_difference/mean": 0.055860452353954315, "step": 353, "step_time": 29.64578586933203 }, { "clip_ratio/high_max": 0.00039443964972936857, "clip_ratio/high_mean": 0.00039443964972936857, "clip_ratio/low_mean": 4.948844555959416e-05, "clip_ratio/low_min": 4.948844555959416e-05, "clip_ratio/region_mean": 0.00044392809892694157, "completions/clipped_ratio": 0.46666669845581055, "completions/max_length": 2048.0, "completions/max_terminated_length": 661.0, "completions/mean_length": 1205.916748046875, "completions/mean_terminated_length": 469.09375, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 9.03132971127828, "epoch": 0.5160349854227405, "frac_reward_zero_std": 0.0, "grad_norm": 0.04770217835903168, "learning_rate": 2e-05, "loss": 0.21852076053619385, "num_tokens": 25873827.0, "reward": -0.32215985655784607, "reward_std": 0.7438012361526489, "rewards/correctness/mean": 0.2666666805744171, "rewards/correctness/std": 0.4459485113620758, "rewards/length_penalty/mean": -0.5888264775276184, "rewards/length_penalty/std": 0.3896961808204651, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9587197303771973, "sampling/importance_sampling_ratio/min": 0.012719303369522095, "sampling/sampling_logp_difference/max": 4.3646345138549805, "sampling/sampling_logp_difference/mean": 0.05479375272989273, "step": 354, "step_time": 28.906675692647696 }, { "clip_ratio/high_max": 0.00027932280257421854, "clip_ratio/high_mean": 0.00027932280257421854, "clip_ratio/low_mean": 7.140815796446986e-05, "clip_ratio/low_min": 7.140815796446986e-05, "clip_ratio/region_mean": 0.00035073095447539043, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 1124.0, "completions/mean_length": 1432.166748046875, "completions/mean_terminated_length": 508.41668701171875, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 9.717802365620932, "epoch": 0.5174927113702624, "frac_reward_zero_std": 0.0, "grad_norm": 0.04212271422147751, "learning_rate": 2e-05, "loss": 0.10804769396781921, "num_tokens": 25964587.0, "reward": -0.41596680879592896, "reward_std": 0.7154483199119568, "rewards/correctness/mean": 0.28333333134651184, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.6993001103401184, "rewards/length_penalty/std": 0.3805501163005829, "sampling/importance_sampling_ratio/max": 2.639859199523926, "sampling/importance_sampling_ratio/mean": 0.9567009806632996, "sampling/importance_sampling_ratio/min": 0.0009637604234740138, "sampling/sampling_logp_difference/max": 6.944667816162109, "sampling/sampling_logp_difference/mean": 0.05627869814634323, "step": 355, "step_time": 29.691745904972777 }, { "clip_ratio/high_max": 0.0003339031245559454, "clip_ratio/high_mean": 0.0003339031245559454, "clip_ratio/low_mean": 3.556317460606806e-05, "clip_ratio/low_min": 3.556317460606806e-05, "clip_ratio/region_mean": 0.00036946630643797107, "completions/clipped_ratio": 0.5833333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 651.0, "completions/mean_length": 1378.1500244140625, "completions/mean_terminated_length": 440.3599853515625, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 9.559980630874634, "epoch": 0.5189504373177842, "frac_reward_zero_std": 0.0, "grad_norm": 0.041803911328315735, "learning_rate": 2e-05, "loss": 0.1954185962677002, "num_tokens": 26051956.0, "reward": -0.5062581896781921, "reward_std": 0.6746494770050049, "rewards/correctness/mean": 0.1666666716337204, "rewards/correctness/std": 0.3758230209350586, "rewards/length_penalty/mean": -0.672924816608429, "rewards/length_penalty/std": 0.39184924960136414, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9568611979484558, "sampling/importance_sampling_ratio/min": 0.09062699973583221, "sampling/sampling_logp_difference/max": 2.401003122329712, "sampling/sampling_logp_difference/mean": 0.0544295571744442, "step": 356, "step_time": 28.6660352780018 }, { "clip_ratio/high_max": 0.00029678063704826246, "clip_ratio/high_mean": 0.00029678063704826246, "clip_ratio/low_mean": 0.00016570400233225277, "clip_ratio/low_min": 0.00016570400233225277, "clip_ratio/region_mean": 0.00046248464786913246, "completions/clipped_ratio": 0.6333333849906921, "completions/max_length": 2048.0, "completions/max_terminated_length": 1511.0, "completions/mean_length": 1483.4334716796875, "completions/mean_terminated_length": 508.2727355957031, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 10.052193323771158, "epoch": 0.5204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.06205817684531212, "learning_rate": 2e-05, "loss": 0.19805066287517548, "num_tokens": 26147512.0, "reward": -0.40766602754592896, "reward_std": 0.800960123538971, "rewards/correctness/mean": 0.3166666626930237, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.7243326902389526, "rewards/length_penalty/std": 0.37654030323028564, "sampling/importance_sampling_ratio/max": 2.7205615043640137, "sampling/importance_sampling_ratio/mean": 0.9547356367111206, "sampling/importance_sampling_ratio/min": 0.0002456428192090243, "sampling/sampling_logp_difference/max": 8.31163215637207, "sampling/sampling_logp_difference/mean": 0.057010468095541, "step": 357, "step_time": 29.62197333574295 }, { "clip_ratio/high_max": 0.0005154515468651274, "clip_ratio/high_mean": 0.0005154515468651274, "clip_ratio/low_mean": 8.461981875977169e-05, "clip_ratio/low_min": 8.461981875977169e-05, "clip_ratio/region_mean": 0.0006000713498603242, "completions/clipped_ratio": 0.36666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1096.0, "completions/mean_length": 939.7667236328125, "completions/mean_terminated_length": 298.15789794921875, "completions/min_length": 114.0, "completions/min_terminated_length": 114.0, "entropy": 9.469589312871298, "epoch": 0.521865889212828, "frac_reward_zero_std": 0.0, "grad_norm": 0.047396935522556305, "learning_rate": 2e-05, "loss": 0.26195502281188965, "num_tokens": 26207728.0, "reward": -0.00887044332921505, "reward_std": 0.8352874517440796, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.45887044072151184, "rewards/length_penalty/std": 0.41956770420074463, "sampling/importance_sampling_ratio/max": 2.146111488342285, "sampling/importance_sampling_ratio/mean": 0.9560447931289673, "sampling/importance_sampling_ratio/min": 0.005923599936068058, "sampling/sampling_logp_difference/max": 5.128810882568359, "sampling/sampling_logp_difference/mean": 0.05592671036720276, "step": 358, "step_time": 26.98090248252265 }, { "clip_ratio/high_max": 0.00033634395852762583, "clip_ratio/high_mean": 0.00033634395852762583, "clip_ratio/low_mean": 0.00018821867464187866, "clip_ratio/low_min": 0.00018821867464187866, "clip_ratio/region_mean": 0.0005245626234682277, "completions/clipped_ratio": 0.4833333492279053, "completions/max_length": 2048.0, "completions/max_terminated_length": 1125.0, "completions/mean_length": 1214.61669921875, "completions/mean_terminated_length": 435.0, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 9.063055992126465, "epoch": 0.5233236151603499, "frac_reward_zero_std": 0.0, "grad_norm": 0.04150667041540146, "learning_rate": 2e-05, "loss": 0.17415601015090942, "num_tokens": 26284725.0, "reward": -0.30974122881889343, "reward_std": 0.7465101480484009, "rewards/correctness/mean": 0.28333333134651184, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.5930745601654053, "rewards/length_penalty/std": 0.400593101978302, "sampling/importance_sampling_ratio/max": 2.038484811782837, "sampling/importance_sampling_ratio/mean": 0.9587417840957642, "sampling/importance_sampling_ratio/min": 0.01532736700028181, "sampling/sampling_logp_difference/max": 4.178115367889404, "sampling/sampling_logp_difference/mean": 0.053427550941705704, "step": 359, "step_time": 27.61032004887238 }, { "clip_ratio/high_max": 0.00012905309995403513, "clip_ratio/high_mean": 0.00012905309995403513, "clip_ratio/low_mean": 0.00012880821426127417, "clip_ratio/low_min": 0.00012880821426127417, "clip_ratio/region_mean": 0.0002578613239165861, "completions/clipped_ratio": 0.4333333671092987, "completions/max_length": 2048.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 1091.5833740234375, "completions/mean_terminated_length": 360.20587158203125, "completions/min_length": 147.0, "completions/min_terminated_length": 147.0, "entropy": 9.084734042485556, "epoch": 0.5247813411078717, "frac_reward_zero_std": 0.0, "grad_norm": 0.04853605479001999, "learning_rate": 2e-05, "loss": 0.257240891456604, "num_tokens": 26353700.0, "reward": -0.06633301079273224, "reward_std": 0.8783034682273865, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.5329996943473816, "rewards/length_penalty/std": 0.41587093472480774, "sampling/importance_sampling_ratio/max": 2.4592902660369873, "sampling/importance_sampling_ratio/mean": 0.9570191502571106, "sampling/importance_sampling_ratio/min": 0.003962638787925243, "sampling/sampling_logp_difference/max": 5.5308451652526855, "sampling/sampling_logp_difference/mean": 0.05573967844247818, "step": 360, "step_time": 27.774955585831776 }, { "clip_ratio/high_max": 0.00041428856396426755, "clip_ratio/high_mean": 0.00041428856396426755, "clip_ratio/low_mean": 0.00010545397890382446, "clip_ratio/low_min": 0.00010545397890382446, "clip_ratio/region_mean": 0.0005197425440807516, "completions/clipped_ratio": 0.3500000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 577.0, "completions/mean_length": 904.4500732421875, "completions/mean_terminated_length": 288.69232177734375, "completions/min_length": 173.0, "completions/min_terminated_length": 173.0, "entropy": 9.182679573694864, "epoch": 0.5262390670553936, "frac_reward_zero_std": 0.0, "grad_norm": 0.04748281091451645, "learning_rate": 2e-05, "loss": 0.16614918410778046, "num_tokens": 26412377.0, "reward": -0.05829264596104622, "reward_std": 0.7925937175750732, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.4416259825229645, "rewards/length_penalty/std": 0.41554591059684753, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9579696655273438, "sampling/importance_sampling_ratio/min": 0.29025036096572876, "sampling/sampling_logp_difference/max": 1.288142442703247, "sampling/sampling_logp_difference/mean": 0.054914653301239014, "step": 361, "step_time": 26.418349273269996 }, { "clip_ratio/high_max": 0.0004512912673817482, "clip_ratio/high_mean": 0.0004512912673817482, "clip_ratio/low_mean": 8.554024316254072e-05, "clip_ratio/low_min": 8.554024316254072e-05, "clip_ratio/region_mean": 0.0005368315129696081, "completions/clipped_ratio": 0.46666669845581055, "completions/max_length": 2048.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 1113.916748046875, "completions/mean_terminated_length": 296.59375, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 9.627775351206461, "epoch": 0.5276967930029155, "frac_reward_zero_std": 0.0, "grad_norm": 0.04629926010966301, "learning_rate": 2e-05, "loss": 0.16687455773353577, "num_tokens": 26485172.0, "reward": -0.11057129502296448, "reward_std": 0.8853543400764465, "rewards/correctness/mean": 0.4333333373069763, "rewards/correctness/std": 0.49971747398376465, "rewards/length_penalty/mean": -0.5439046025276184, "rewards/length_penalty/std": 0.43338432908058167, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9573235511779785, "sampling/importance_sampling_ratio/min": 0.20859287679195404, "sampling/sampling_logp_difference/max": 1.567370891571045, "sampling/sampling_logp_difference/mean": 0.05558919161558151, "step": 362, "step_time": 29.222585463896394 }, { "clip_ratio/high_max": 0.0002973089831357356, "clip_ratio/high_mean": 0.0002973089831357356, "clip_ratio/low_mean": 0.00015872955433830308, "clip_ratio/low_min": 0.00015872955433830308, "clip_ratio/region_mean": 0.0004560385229221235, "completions/clipped_ratio": 0.5333333611488342, "completions/max_length": 2048.0, "completions/max_terminated_length": 690.0, "completions/mean_length": 1271.4000244140625, "completions/mean_terminated_length": 383.8571472167969, "completions/min_length": 99.0, "completions/min_terminated_length": 99.0, "entropy": 9.186814387639364, "epoch": 0.5291545189504373, "frac_reward_zero_std": 0.0, "grad_norm": 0.04732315614819527, "learning_rate": 2e-05, "loss": 0.19950109720230103, "num_tokens": 26566806.0, "reward": -0.3208008110523224, "reward_std": 0.8106894493103027, "rewards/correctness/mean": 0.30000001192092896, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.620800793170929, "rewards/length_penalty/std": 0.4124334752559662, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9574229121208191, "sampling/importance_sampling_ratio/min": 0.014777660369873047, "sampling/sampling_logp_difference/max": 4.214638710021973, "sampling/sampling_logp_difference/mean": 0.05504994094371796, "step": 363, "step_time": 29.084793600719422 }, { "clip_ratio/high_max": 0.00033026450061394524, "clip_ratio/high_mean": 0.00033026450061394524, "clip_ratio/low_mean": 0.00010910878215023938, "clip_ratio/low_min": 0.00010910878215023938, "clip_ratio/region_mean": 0.00043937327185024816, "completions/clipped_ratio": 0.5, "completions/max_length": 2048.0, "completions/max_terminated_length": 854.0, "completions/mean_length": 1267.4000244140625, "completions/mean_terminated_length": 486.8000183105469, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 8.936376810073853, "epoch": 0.5306122448979592, "frac_reward_zero_std": 0.0, "grad_norm": 0.04914522171020508, "learning_rate": 2e-05, "loss": 0.15480926632881165, "num_tokens": 26647220.0, "reward": -0.08551432937383652, "reward_std": 0.8274834156036377, "rewards/correctness/mean": 0.5333333611488342, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.618847668170929, "rewards/length_penalty/std": 0.3896554112434387, "sampling/importance_sampling_ratio/max": 2.8322601318359375, "sampling/importance_sampling_ratio/mean": 0.9595205187797546, "sampling/importance_sampling_ratio/min": 0.15685983002185822, "sampling/sampling_logp_difference/max": 1.852402687072754, "sampling/sampling_logp_difference/mean": 0.05048660561442375, "step": 364, "step_time": 28.076930057955906 }, { "clip_ratio/high_max": 0.0007259485379715139, "clip_ratio/high_mean": 0.0007259485379715139, "clip_ratio/low_mean": 0.00020760440626569712, "clip_ratio/low_min": 0.00020760440626569712, "clip_ratio/region_mean": 0.0009335529466625303, "completions/clipped_ratio": 0.31666669249534607, "completions/max_length": 2048.0, "completions/max_terminated_length": 604.0, "completions/mean_length": 892.6500244140625, "completions/mean_terminated_length": 357.243896484375, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 8.23016881942749, "epoch": 0.532069970845481, "frac_reward_zero_std": 0.0, "grad_norm": 0.06116010248661041, "learning_rate": 2e-05, "loss": 0.20466528832912445, "num_tokens": 26704249.0, "reward": 0.0308024100959301, "reward_std": 0.7972607016563416, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.43586426973342896, "rewards/length_penalty/std": 0.3886968195438385, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9626304507255554, "sampling/importance_sampling_ratio/min": 0.04832625016570091, "sampling/sampling_logp_difference/max": 3.029780387878418, "sampling/sampling_logp_difference/mean": 0.04985644295811653, "step": 365, "step_time": 26.8528596656397 }, { "clip_ratio/high_max": 0.0004172012201403656, "clip_ratio/high_mean": 0.0004172012201403656, "clip_ratio/low_mean": 7.514922738967773e-05, "clip_ratio/low_min": 7.514922738967773e-05, "clip_ratio/region_mean": 0.0004923504505616924, "completions/clipped_ratio": 0.5666667222976685, "completions/max_length": 2048.0, "completions/max_terminated_length": 569.0, "completions/mean_length": 1304.666748046875, "completions/mean_terminated_length": 332.6153869628906, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 10.152477264404297, "epoch": 0.5335276967930029, "frac_reward_zero_std": 0.0, "grad_norm": 0.04620922729372978, "learning_rate": 2e-05, "loss": 0.1625717431306839, "num_tokens": 26791389.0, "reward": -0.37037762999534607, "reward_std": 0.7341510057449341, "rewards/correctness/mean": 0.2666666805744171, "rewards/correctness/std": 0.4459485113620758, "rewards/length_penalty/mean": -0.6370442509651184, "rewards/length_penalty/std": 0.4204311966896057, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.954558789730072, "sampling/importance_sampling_ratio/min": 0.028885751962661743, "sampling/sampling_logp_difference/max": 3.5444068908691406, "sampling/sampling_logp_difference/mean": 0.05660395324230194, "step": 366, "step_time": 29.953183996025473 }, { "clip_ratio/high_max": 0.00017920291429618374, "clip_ratio/high_mean": 0.00017920291429618374, "clip_ratio/low_mean": 0.00017146518560669696, "clip_ratio/low_min": 0.00017146518560669696, "clip_ratio/region_mean": 0.0003506681047535191, "completions/clipped_ratio": 0.6333333849906921, "completions/max_length": 2048.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 1472.283447265625, "completions/mean_terminated_length": 477.8636474609375, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 9.43461799621582, "epoch": 0.5349854227405247, "frac_reward_zero_std": 0.0, "grad_norm": 0.03412831202149391, "learning_rate": 2e-05, "loss": 0.1196075901389122, "num_tokens": 26889356.0, "reward": -0.6355550289154053, "reward_std": 0.5291460752487183, "rewards/correctness/mean": 0.0833333358168602, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.7188883423805237, "rewards/length_penalty/std": 0.38189852237701416, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9574069380760193, "sampling/importance_sampling_ratio/min": 0.04516036808490753, "sampling/sampling_logp_difference/max": 3.0975353717803955, "sampling/sampling_logp_difference/mean": 0.05276482179760933, "step": 367, "step_time": 31.844728611409664 }, { "clip_ratio/high_max": 0.0004386513343585345, "clip_ratio/high_mean": 0.0004386513343585345, "clip_ratio/low_mean": 0.00021285747061483562, "clip_ratio/low_min": 0.00021285747061483562, "clip_ratio/region_mean": 0.0006515088170999661, "completions/clipped_ratio": 0.4333333671092987, "completions/max_length": 2048.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 1123.4833984375, "completions/mean_terminated_length": 416.5, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 9.059595266977945, "epoch": 0.5364431486880467, "frac_reward_zero_std": 0.0, "grad_norm": 0.035539690405130386, "learning_rate": 2e-05, "loss": 0.2148118019104004, "num_tokens": 26960555.0, "reward": -0.29857584834098816, "reward_std": 0.7376112341880798, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.5485758185386658, "rewards/length_penalty/std": 0.4032915532588959, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9582176208496094, "sampling/importance_sampling_ratio/min": 0.06377708911895752, "sampling/sampling_logp_difference/max": 2.752361297607422, "sampling/sampling_logp_difference/mean": 0.05178624391555786, "step": 368, "step_time": 28.118968724040315 }, { "clip_ratio/high_max": 0.0006257997447391972, "clip_ratio/high_mean": 0.0006257997447391972, "clip_ratio/low_mean": 0.0001840256760867002, "clip_ratio/low_min": 0.0001840256760867002, "clip_ratio/region_mean": 0.000809825403848663, "completions/clipped_ratio": 0.46666669845581055, "completions/max_length": 2048.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 1133.9500732421875, "completions/mean_terminated_length": 334.15625, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 8.871262709299723, "epoch": 0.5379008746355685, "frac_reward_zero_std": 0.0, "grad_norm": 0.048182569444179535, "learning_rate": 2e-05, "loss": 0.1919139325618744, "num_tokens": 27032992.0, "reward": -0.17035320401191711, "reward_std": 0.8425008058547974, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.5536864995956421, "rewards/length_penalty/std": 0.42298001050949097, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9594973921775818, "sampling/importance_sampling_ratio/min": 0.02479230985045433, "sampling/sampling_logp_difference/max": 3.6972217559814453, "sampling/sampling_logp_difference/mean": 0.05171066150069237, "step": 369, "step_time": 27.715583306970075 }, { "clip_ratio/high_max": 0.0006179433791354919, "clip_ratio/high_mean": 0.0006179433791354919, "clip_ratio/low_mean": 0.0001227502349744706, "clip_ratio/low_min": 0.0001227502349744706, "clip_ratio/region_mean": 0.0007406936201732606, "completions/clipped_ratio": 0.36666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1350.0, "completions/mean_length": 1086.4500732421875, "completions/mean_terminated_length": 529.76318359375, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 7.8825608889261884, "epoch": 0.5393586005830904, "frac_reward_zero_std": 0.0, "grad_norm": 0.04518386349081993, "learning_rate": 2e-05, "loss": 0.14720721542835236, "num_tokens": 27102499.0, "reward": -0.26382651925086975, "reward_std": 0.7128307819366455, "rewards/correctness/mean": 0.2666666805744171, "rewards/correctness/std": 0.4459485113620758, "rewards/length_penalty/mean": -0.5304931402206421, "rewards/length_penalty/std": 0.3813227713108063, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9642049074172974, "sampling/importance_sampling_ratio/min": 0.07521047443151474, "sampling/sampling_logp_difference/max": 2.5874648094177246, "sampling/sampling_logp_difference/mean": 0.045135945081710815, "step": 370, "step_time": 28.29425092902966 }, { "clip_ratio/high_max": 0.00019403680077327104, "clip_ratio/high_mean": 0.00019403680077327104, "clip_ratio/low_mean": 0.00023081649123923853, "clip_ratio/low_min": 0.00023081649123923853, "clip_ratio/region_mean": 0.0004248532883745308, "completions/clipped_ratio": 0.6000000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 1271.0, "completions/mean_length": 1439.4000244140625, "completions/mean_terminated_length": 526.5, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 9.005580504735311, "epoch": 0.5408163265306123, "frac_reward_zero_std": 0.0, "grad_norm": 0.03809776157140732, "learning_rate": 2e-05, "loss": 0.1817202866077423, "num_tokens": 27195073.0, "reward": -0.5028320550918579, "reward_std": 0.6847959160804749, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.702832043170929, "rewards/length_penalty/std": 0.37536630034446716, "sampling/importance_sampling_ratio/max": 2.6483681201934814, "sampling/importance_sampling_ratio/mean": 0.9589487314224243, "sampling/importance_sampling_ratio/min": 0.0018576967995613813, "sampling/sampling_logp_difference/max": 6.288417816162109, "sampling/sampling_logp_difference/mean": 0.050450630486011505, "step": 371, "step_time": 29.88444741629064 }, { "clip_ratio/high_max": 0.00032072564742217463, "clip_ratio/high_mean": 0.00032072564742217463, "clip_ratio/low_mean": 0.00023663697114291912, "clip_ratio/low_min": 0.00023663697114291912, "clip_ratio/region_mean": 0.0005573626306916898, "completions/clipped_ratio": 0.5, "completions/max_length": 2048.0, "completions/max_terminated_length": 615.0, "completions/mean_length": 1200.6334228515625, "completions/mean_terminated_length": 353.2666931152344, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 9.506787697474161, "epoch": 0.5422740524781341, "frac_reward_zero_std": 0.0, "grad_norm": 0.04333457723259926, "learning_rate": 2e-05, "loss": 0.25675034523010254, "num_tokens": 27270961.0, "reward": -0.21958008408546448, "reward_std": 0.8319997191429138, "rewards/correctness/mean": 0.36666667461395264, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.5862467288970947, "rewards/length_penalty/std": 0.4182293117046356, "sampling/importance_sampling_ratio/max": 2.337223529815674, "sampling/importance_sampling_ratio/mean": 0.9562131762504578, "sampling/importance_sampling_ratio/min": 0.09544428437948227, "sampling/sampling_logp_difference/max": 2.349212646484375, "sampling/sampling_logp_difference/mean": 0.053766850382089615, "step": 372, "step_time": 27.871957514202222 }, { "clip_ratio/high_max": 0.0005797006306238472, "clip_ratio/high_mean": 0.0005797006306238472, "clip_ratio/low_mean": 0.00015036369344064346, "clip_ratio/low_min": 0.00015036369344064346, "clip_ratio/region_mean": 0.000730064338616406, "completions/clipped_ratio": 0.45000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 643.0, "completions/mean_length": 1089.033447265625, "completions/mean_terminated_length": 304.42425537109375, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 9.004942258199057, "epoch": 0.543731778425656, "frac_reward_zero_std": 0.0, "grad_norm": 0.04386036843061447, "learning_rate": 2e-05, "loss": 0.15573643147945404, "num_tokens": 27341343.0, "reward": -0.28175458312034607, "reward_std": 0.769770085811615, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.5317545533180237, "rewards/length_penalty/std": 0.4298044443130493, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9572499990463257, "sampling/importance_sampling_ratio/min": 0.03063114359974861, "sampling/sampling_logp_difference/max": 3.4857380390167236, "sampling/sampling_logp_difference/mean": 0.05183059722185135, "step": 373, "step_time": 28.341473578475416 }, { "clip_ratio/high_max": 0.0007873279428167734, "clip_ratio/high_mean": 0.0007873279428167734, "clip_ratio/low_mean": 0.00017677067201778604, "clip_ratio/low_min": 0.00017677067201778604, "clip_ratio/region_mean": 0.0009640986075586019, "completions/clipped_ratio": 0.31666669249534607, "completions/max_length": 2048.0, "completions/max_terminated_length": 496.0, "completions/mean_length": 837.0833740234375, "completions/mean_terminated_length": 275.92681884765625, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "entropy": 8.190476814905802, "epoch": 0.5451895043731778, "frac_reward_zero_std": 0.0, "grad_norm": 0.046187106519937515, "learning_rate": 2e-05, "loss": 0.2520688474178314, "num_tokens": 27395658.0, "reward": 0.24126791954040527, "reward_std": 0.8732016086578369, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.4087320864200592, "rewards/length_penalty/std": 0.40733250975608826, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9598665237426758, "sampling/importance_sampling_ratio/min": 0.061528608202934265, "sampling/sampling_logp_difference/max": 2.78825306892395, "sampling/sampling_logp_difference/mean": 0.05003877729177475, "step": 374, "step_time": 26.110598615836352 }, { "clip_ratio/high_max": 0.0005037667918562269, "clip_ratio/high_mean": 0.0005037667918562269, "clip_ratio/low_mean": 0.00021512723954704901, "clip_ratio/low_min": 0.00021512723954704901, "clip_ratio/region_mean": 0.0007188940362539142, "completions/clipped_ratio": 0.46666669845581055, "completions/max_length": 2048.0, "completions/max_terminated_length": 803.0, "completions/mean_length": 1178.35009765625, "completions/mean_terminated_length": 417.40625, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "entropy": 8.922356605529785, "epoch": 0.5466472303206997, "frac_reward_zero_std": 0.0, "grad_norm": 0.04022344574332237, "learning_rate": 2e-05, "loss": 0.1425526738166809, "num_tokens": 27470679.0, "reward": -0.2753662168979645, "reward_std": 0.8021130561828613, "rewards/correctness/mean": 0.30000001192092896, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.575366199016571, "rewards/length_penalty/std": 0.4079590439796448, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9596368074417114, "sampling/importance_sampling_ratio/min": 0.004272452089935541, "sampling/sampling_logp_difference/max": 5.455567359924316, "sampling/sampling_logp_difference/mean": 0.050488341599702835, "step": 375, "step_time": 27.60874852631241 }, { "clip_ratio/high_max": 0.0003316489407249416, "clip_ratio/high_mean": 0.0003316489407249416, "clip_ratio/low_mean": 0.00011301313255292673, "clip_ratio/low_min": 0.00011301313255292673, "clip_ratio/region_mean": 0.00044466207327786833, "completions/clipped_ratio": 0.5333333611488342, "completions/max_length": 2048.0, "completions/max_terminated_length": 873.0, "completions/mean_length": 1304.8834228515625, "completions/mean_terminated_length": 455.607177734375, "completions/min_length": 140.0, "completions/min_terminated_length": 140.0, "entropy": 8.77918028831482, "epoch": 0.5481049562682215, "frac_reward_zero_std": 0.0, "grad_norm": 0.04689865559339523, "learning_rate": 2e-05, "loss": 0.29697903990745544, "num_tokens": 27554532.0, "reward": -0.6371501088142395, "reward_std": 0.39772436022758484, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.6371500492095947, "rewards/length_penalty/std": 0.39772436022758484, "sampling/importance_sampling_ratio/max": 2.354552984237671, "sampling/importance_sampling_ratio/mean": 0.96060711145401, "sampling/importance_sampling_ratio/min": 0.09059971570968628, "sampling/sampling_logp_difference/max": 2.401304244995117, "sampling/sampling_logp_difference/mean": 0.04888860881328583, "step": 376, "step_time": 29.419202062767 }, { "clip_ratio/high_max": 0.0004210337550224115, "clip_ratio/high_mean": 0.0004210337550224115, "clip_ratio/low_mean": 0.0001944365406719347, "clip_ratio/low_min": 0.0001944365406719347, "clip_ratio/region_mean": 0.0006154702956943462, "completions/clipped_ratio": 0.4333333671092987, "completions/max_length": 2048.0, "completions/max_terminated_length": 1503.0, "completions/mean_length": 1221.783447265625, "completions/mean_terminated_length": 589.9705810546875, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 7.848694324493408, "epoch": 0.5495626822157434, "frac_reward_zero_std": 0.0, "grad_norm": 0.0429447703063488, "learning_rate": 2e-05, "loss": 0.26736709475517273, "num_tokens": 27634109.0, "reward": -0.21324056386947632, "reward_std": 0.8117257952690125, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.5965738892555237, "rewards/length_penalty/std": 0.37753546237945557, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9643316864967346, "sampling/importance_sampling_ratio/min": 0.05885998159646988, "sampling/sampling_logp_difference/max": 2.8325939178466797, "sampling/sampling_logp_difference/mean": 0.045014638453722, "step": 377, "step_time": 28.57699161604978 }, { "clip_ratio/high_max": 0.0005955963885450425, "clip_ratio/high_mean": 0.0005955963885450425, "clip_ratio/low_mean": 0.0002728259411621063, "clip_ratio/low_min": 0.0002728259411621063, "clip_ratio/region_mean": 0.000868422289689382, "completions/clipped_ratio": 0.40000003576278687, "completions/max_length": 2048.0, "completions/max_terminated_length": 889.0, "completions/mean_length": 1044.0833740234375, "completions/mean_terminated_length": 374.8055725097656, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 9.009823083877563, "epoch": 0.5510204081632653, "frac_reward_zero_std": 0.0, "grad_norm": 0.04480404034256935, "learning_rate": 2e-05, "loss": 0.11866474151611328, "num_tokens": 27701064.0, "reward": -0.15980632603168488, "reward_std": 0.8080062866210938, "rewards/correctness/mean": 0.3499999940395355, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.5098063349723816, "rewards/length_penalty/std": 0.4090655744075775, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9598076939582825, "sampling/importance_sampling_ratio/min": 0.001706836512312293, "sampling/sampling_logp_difference/max": 6.373113632202148, "sampling/sampling_logp_difference/mean": 0.05107296258211136, "step": 378, "step_time": 27.93185718660243 }, { "clip_ratio/high_max": 0.0006190764470375143, "clip_ratio/high_mean": 0.0006190764470375143, "clip_ratio/low_mean": 0.0002111473028586867, "clip_ratio/low_min": 0.0002111473028586867, "clip_ratio/region_mean": 0.0008302237353442857, "completions/clipped_ratio": 0.4166666865348816, "completions/max_length": 2048.0, "completions/max_terminated_length": 876.0, "completions/mean_length": 1061.166748046875, "completions/mean_terminated_length": 356.28570556640625, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 9.222933371861776, "epoch": 0.5524781341107872, "frac_reward_zero_std": 0.0, "grad_norm": 0.04446539282798767, "learning_rate": 2e-05, "loss": 0.17164865136146545, "num_tokens": 27772034.0, "reward": -0.16814780235290527, "reward_std": 0.7861822247505188, "rewards/correctness/mean": 0.3499999940395355, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.5181477665901184, "rewards/length_penalty/std": 0.4147971272468567, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9586564302444458, "sampling/importance_sampling_ratio/min": 0.029254186898469925, "sampling/sampling_logp_difference/max": 3.5317325592041016, "sampling/sampling_logp_difference/mean": 0.053076643496751785, "step": 379, "step_time": 29.396459304727614 }, { "clip_ratio/high_max": 0.0006856544205220416, "clip_ratio/high_mean": 0.0006856544205220416, "clip_ratio/low_mean": 0.0003764652186267388, "clip_ratio/low_min": 0.0003764652186267388, "clip_ratio/region_mean": 0.0010621196318728228, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1186.0, "completions/mean_length": 730.9833984375, "completions/mean_terminated_length": 401.72918701171875, "completions/min_length": 194.0, "completions/min_terminated_length": 194.0, "entropy": 6.719113906224568, "epoch": 0.5539358600583091, "frac_reward_zero_std": 0.0, "grad_norm": 0.04761587083339691, "learning_rate": 2e-05, "loss": 0.14682242274284363, "num_tokens": 27820043.0, "reward": 0.24307455122470856, "reward_std": 0.7441136240959167, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.35692545771598816, "rewards/length_penalty/std": 0.33469006419181824, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.968551754951477, "sampling/importance_sampling_ratio/min": 0.001233525574207306, "sampling/sampling_logp_difference/max": 6.697878837585449, "sampling/sampling_logp_difference/mean": 0.04193832352757454, "step": 380, "step_time": 25.490312166977674 }, { "clip_ratio/high_max": 0.0006712004784882689, "clip_ratio/high_mean": 0.0006712004784882689, "clip_ratio/low_mean": 0.00021278198861788647, "clip_ratio/low_min": 0.00021278198861788647, "clip_ratio/region_mean": 0.0008839824731694534, "completions/clipped_ratio": 0.3500000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 834.0, "completions/mean_length": 994.1000366210938, "completions/mean_terminated_length": 426.6153869628906, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 8.234848101933798, "epoch": 0.5553935860058309, "frac_reward_zero_std": 0.0, "grad_norm": 0.053330808877944946, "learning_rate": 2e-05, "loss": 0.21266715228557587, "num_tokens": 27885659.0, "reward": -0.035400390625, "reward_std": 0.8141936659812927, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.48540037870407104, "rewards/length_penalty/std": 0.38425105810165405, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.962712287902832, "sampling/importance_sampling_ratio/min": 0.002823264105245471, "sampling/sampling_logp_difference/max": 5.869861602783203, "sampling/sampling_logp_difference/mean": 0.047032348811626434, "step": 381, "step_time": 28.263548829127103 }, { "clip_ratio/high_max": 0.0012803510520219181, "clip_ratio/high_mean": 0.0012803510520219181, "clip_ratio/low_mean": 0.00024391300394199789, "clip_ratio/low_min": 0.00024391300394199789, "clip_ratio/region_mean": 0.001524264089918385, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 864.0, "completions/mean_length": 779.4833984375, "completions/mean_terminated_length": 462.35418701171875, "completions/min_length": 154.0, "completions/min_terminated_length": 154.0, "entropy": 6.00438638528188, "epoch": 0.5568513119533528, "frac_reward_zero_std": 0.0, "grad_norm": 0.049449775367975235, "learning_rate": 2e-05, "loss": 0.19709795713424683, "num_tokens": 27936398.0, "reward": 0.23605957627296448, "reward_std": 0.7617997527122498, "rewards/correctness/mean": 0.6166666746139526, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.38060709834098816, "rewards/length_penalty/std": 0.3286638557910919, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9709339737892151, "sampling/importance_sampling_ratio/min": 0.09268508851528168, "sampling/sampling_logp_difference/max": 2.3785476684570312, "sampling/sampling_logp_difference/mean": 0.039329078048467636, "step": 382, "step_time": 25.73606661008671 }, { "clip_ratio/high_max": 0.0006209056421842737, "clip_ratio/high_mean": 0.0006209056421842737, "clip_ratio/low_mean": 0.0001912688918916198, "clip_ratio/low_min": 0.0001912688918916198, "clip_ratio/region_mean": 0.000812174529225255, "completions/clipped_ratio": 0.36666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1131.0, "completions/mean_length": 1009.6500244140625, "completions/mean_terminated_length": 408.5, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 8.126364866892496, "epoch": 0.5583090379008746, "frac_reward_zero_std": 0.0, "grad_norm": 0.04511874541640282, "learning_rate": 2e-05, "loss": 0.23375315964221954, "num_tokens": 28001177.0, "reward": 0.0570068396627903, "reward_std": 0.8558456897735596, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.49299317598342896, "rewards/length_penalty/std": 0.3951627016067505, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9625617861747742, "sampling/importance_sampling_ratio/min": 0.004374358803033829, "sampling/sampling_logp_difference/max": 5.431995391845703, "sampling/sampling_logp_difference/mean": 0.0455813854932785, "step": 383, "step_time": 27.085484920069575 }, { "clip_ratio/high_max": 0.0006755723152309656, "clip_ratio/high_mean": 0.0006755723152309656, "clip_ratio/low_mean": 0.00014645593546447344, "clip_ratio/low_min": 0.00014645593546447344, "clip_ratio/region_mean": 0.0008220282761612907, "completions/clipped_ratio": 0.3333333432674408, "completions/max_length": 2048.0, "completions/max_terminated_length": 1331.0, "completions/mean_length": 990.3833618164062, "completions/mean_terminated_length": 461.57501220703125, "completions/min_length": 154.0, "completions/min_terminated_length": 154.0, "entropy": 7.575742085774739, "epoch": 0.5597667638483965, "frac_reward_zero_std": 0.0, "grad_norm": 0.043413110077381134, "learning_rate": 2e-05, "loss": 0.16492630541324615, "num_tokens": 28065820.0, "reward": -0.08358561992645264, "reward_std": 0.8001952767372131, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49403220415115356, "rewards/length_penalty/mean": -0.4835856258869171, "rewards/length_penalty/std": 0.38215550780296326, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9646731019020081, "sampling/importance_sampling_ratio/min": 0.09302797168493271, "sampling/sampling_logp_difference/max": 2.3748550415039062, "sampling/sampling_logp_difference/mean": 0.04558727890253067, "step": 384, "step_time": 27.539941049879417 }, { "clip_ratio/high_max": 0.0009427356950861091, "clip_ratio/high_mean": 0.0009427356950861091, "clip_ratio/low_mean": 0.00026784483998198994, "clip_ratio/low_min": 0.00026784483998198994, "clip_ratio/region_mean": 0.001210580550832674, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 1000.6834106445312, "completions/mean_terminated_length": 619.8409423828125, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 6.0675002336502075, "epoch": 0.5612244897959183, "frac_reward_zero_std": 0.0, "grad_norm": 0.042015690356492996, "learning_rate": 2e-05, "loss": 0.14073562622070312, "num_tokens": 28133731.0, "reward": -0.23861491680145264, "reward_std": 0.6667718887329102, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.48861491680145264, "rewards/length_penalty/std": 0.3250850737094879, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9717772006988525, "sampling/importance_sampling_ratio/min": 0.040409915149211884, "sampling/sampling_logp_difference/max": 3.2086801528930664, "sampling/sampling_logp_difference/mean": 0.0385608896613121, "step": 385, "step_time": 27.7715870779939 }, { "clip_ratio/high_max": 0.0008884885416288549, "clip_ratio/high_mean": 0.0008884885416288549, "clip_ratio/low_mean": 0.00015518833121556477, "clip_ratio/low_min": 0.00015518833121556477, "clip_ratio/region_mean": 0.001043676893459633, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 2048.0, "completions/max_terminated_length": 792.0, "completions/mean_length": 891.300048828125, "completions/mean_terminated_length": 395.5714416503906, "completions/min_length": 167.0, "completions/min_terminated_length": 167.0, "entropy": 7.474557558695476, "epoch": 0.5626822157434402, "frac_reward_zero_std": 0.0, "grad_norm": 0.04013489559292793, "learning_rate": 2e-05, "loss": 0.17641425132751465, "num_tokens": 28191199.0, "reward": -0.13520509004592896, "reward_std": 0.7179872989654541, "rewards/correctness/mean": 0.30000001192092896, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.4352050721645355, "rewards/length_penalty/std": 0.37735384702682495, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.962986409664154, "sampling/importance_sampling_ratio/min": 0.0852149948477745, "sampling/sampling_logp_difference/max": 2.4625778198242188, "sampling/sampling_logp_difference/mean": 0.04626065492630005, "step": 386, "step_time": 26.535623141331598 }, { "clip_ratio/high_max": 0.0002034776371147018, "clip_ratio/high_mean": 0.0002034776371147018, "clip_ratio/low_mean": 0.0002736892565735616, "clip_ratio/low_min": 0.0002736892565735616, "clip_ratio/region_mean": 0.0004771668997515614, "completions/clipped_ratio": 0.4166666865348816, "completions/max_length": 2048.0, "completions/max_terminated_length": 952.0, "completions/mean_length": 1066.2166748046875, "completions/mean_terminated_length": 364.94287109375, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 8.900734027226767, "epoch": 0.5641399416909622, "frac_reward_zero_std": 0.0, "grad_norm": 0.04746143892407417, "learning_rate": 2e-05, "loss": 0.18093833327293396, "num_tokens": 28259402.0, "reward": -0.05394694209098816, "reward_std": 0.8789756894111633, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.5206136107444763, "rewards/length_penalty/std": 0.41460806131362915, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9598606824874878, "sampling/importance_sampling_ratio/min": 0.0027541639283299446, "sampling/sampling_logp_difference/max": 5.894641399383545, "sampling/sampling_logp_difference/mean": 0.049475423991680145, "step": 387, "step_time": 27.756329206982628 }, { "clip_ratio/high_max": 0.0011696891645745684, "clip_ratio/high_mean": 0.0011696891645745684, "clip_ratio/low_mean": 0.000370938767446205, "clip_ratio/low_min": 0.000370938767446205, "clip_ratio/region_mean": 0.0015406279611246039, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 749.2500610351562, "completions/mean_terminated_length": 520.058837890625, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 4.565074414014816, "epoch": 0.565597667638484, "frac_reward_zero_std": 0.0, "grad_norm": 0.03902987763285637, "learning_rate": 2e-05, "loss": 0.09935897588729858, "num_tokens": 28309307.0, "reward": 0.01748860813677311, "reward_std": 0.6527488827705383, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.3658447265625, "rewards/length_penalty/std": 0.2808268368244171, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9763094782829285, "sampling/importance_sampling_ratio/min": 0.01139597687870264, "sampling/sampling_logp_difference/max": 4.474494934082031, "sampling/sampling_logp_difference/mean": 0.0351274274289608, "step": 388, "step_time": 26.141073181526735 }, { "clip_ratio/high_max": 0.0006787832050273815, "clip_ratio/high_mean": 0.0006787832050273815, "clip_ratio/low_mean": 0.00022614005380698168, "clip_ratio/low_min": 0.00022614005380698168, "clip_ratio/region_mean": 0.0009049232515584057, "completions/clipped_ratio": 0.38333335518836975, "completions/max_length": 2048.0, "completions/max_terminated_length": 860.0, "completions/mean_length": 1009.2667236328125, "completions/mean_terminated_length": 363.56756591796875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 8.777956326802572, "epoch": 0.5670553935860059, "frac_reward_zero_std": 0.0, "grad_norm": 0.04908052459359169, "learning_rate": 2e-05, "loss": 0.23067492246627808, "num_tokens": 28374003.0, "reward": -0.0428059920668602, "reward_std": 0.862494945526123, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.49280598759651184, "rewards/length_penalty/std": 0.4088231921195984, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9601799845695496, "sampling/importance_sampling_ratio/min": 0.0021186680532991886, "sampling/sampling_logp_difference/max": 6.156967639923096, "sampling/sampling_logp_difference/mean": 0.04961955547332764, "step": 389, "step_time": 26.942984137218446 }, { "clip_ratio/high_max": 0.0010587582655716687, "clip_ratio/high_mean": 0.0010587582655716687, "clip_ratio/low_mean": 0.00014047935474081896, "clip_ratio/low_min": 0.00014047935474081896, "clip_ratio/region_mean": 0.0011992376142491896, "completions/clipped_ratio": 0.28333336114883423, "completions/max_length": 2048.0, "completions/max_terminated_length": 876.0, "completions/mean_length": 875.86669921875, "completions/mean_terminated_length": 412.4651184082031, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 7.477418740590413, "epoch": 0.5685131195335277, "frac_reward_zero_std": 0.0, "grad_norm": 0.05427652969956398, "learning_rate": 2e-05, "loss": 0.2583100199699402, "num_tokens": 28430085.0, "reward": 0.20566406846046448, "reward_std": 0.8246890902519226, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.4276692569255829, "rewards/length_penalty/std": 0.36581674218177795, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9648388624191284, "sampling/importance_sampling_ratio/min": 0.05480140820145607, "sampling/sampling_logp_difference/max": 2.9040393829345703, "sampling/sampling_logp_difference/mean": 0.0434843972325325, "step": 390, "step_time": 26.16308691422455 }, { "clip_ratio/high_max": 0.0005601125352162247, "clip_ratio/high_mean": 0.0005601125352162247, "clip_ratio/low_mean": 0.00026296248446063447, "clip_ratio/low_min": 0.00026296248446063447, "clip_ratio/region_mean": 0.000823075029378136, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 2048.0, "completions/max_terminated_length": 801.0, "completions/mean_length": 912.5333862304688, "completions/mean_terminated_length": 425.9047546386719, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 7.8242223262786865, "epoch": 0.5699708454810496, "frac_reward_zero_std": 0.0, "grad_norm": 0.0444527193903923, "learning_rate": 2e-05, "loss": 0.23224171996116638, "num_tokens": 28489427.0, "reward": 0.08776041865348816, "reward_std": 0.8116838932037354, "rewards/correctness/mean": 0.5333333611488342, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.4455729126930237, "rewards/length_penalty/std": 0.36872655153274536, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9645928740501404, "sampling/importance_sampling_ratio/min": 0.0417904295027256, "sampling/sampling_logp_difference/max": 3.1750879287719727, "sampling/sampling_logp_difference/mean": 0.04424896091222763, "step": 391, "step_time": 27.045860501937568 }, { "clip_ratio/high_max": 0.0011545448117734243, "clip_ratio/high_mean": 0.0011545448117734243, "clip_ratio/low_mean": 0.00015252744681977978, "clip_ratio/low_min": 0.00015252744681977978, "clip_ratio/region_mean": 0.0013070722537425656, "completions/clipped_ratio": 0.31666669249534607, "completions/max_length": 2048.0, "completions/max_terminated_length": 1178.0, "completions/mean_length": 1005.11669921875, "completions/mean_terminated_length": 521.8292236328125, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 7.445602575937907, "epoch": 0.5714285714285714, "frac_reward_zero_std": 0.0, "grad_norm": 0.04083642363548279, "learning_rate": 2e-05, "loss": 0.18471992015838623, "num_tokens": 28553844.0, "reward": -0.12411296367645264, "reward_std": 0.7303255200386047, "rewards/correctness/mean": 0.36666667461395264, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.4907796084880829, "rewards/length_penalty/std": 0.36357301473617554, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9647839069366455, "sampling/importance_sampling_ratio/min": 0.025366825982928276, "sampling/sampling_logp_difference/max": 3.6743130683898926, "sampling/sampling_logp_difference/mean": 0.044303543865680695, "step": 392, "step_time": 26.763385435566306 }, { "clip_ratio/high_max": 0.000623729692354876, "clip_ratio/high_mean": 0.000623729692354876, "clip_ratio/low_mean": 0.000284906924207462, "clip_ratio/low_min": 0.000284906924207462, "clip_ratio/region_mean": 0.0009086366068610611, "completions/clipped_ratio": 0.36666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1312.0, "completions/mean_length": 1088.3834228515625, "completions/mean_terminated_length": 532.8157958984375, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 7.401313066482544, "epoch": 0.5728862973760933, "frac_reward_zero_std": 0.0, "grad_norm": 0.05181447044014931, "learning_rate": 2e-05, "loss": 0.18945720791816711, "num_tokens": 28625277.0, "reward": 0.0018961589084938169, "reward_std": 0.8546024560928345, "rewards/correctness/mean": 0.5333333611488342, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.5314371585845947, "rewards/length_penalty/std": 0.37617993354797363, "sampling/importance_sampling_ratio/max": 2.8986191749572754, "sampling/importance_sampling_ratio/mean": 0.9640436172485352, "sampling/importance_sampling_ratio/min": 0.050960954278707504, "sampling/sampling_logp_difference/max": 2.9766955375671387, "sampling/sampling_logp_difference/mean": 0.0436813123524189, "step": 393, "step_time": 27.83029830805026 }, { "clip_ratio/high_max": 0.0006723449951095972, "clip_ratio/high_mean": 0.0006723449951095972, "clip_ratio/low_mean": 0.00014831762625059733, "clip_ratio/low_min": 0.00014831762625059733, "clip_ratio/region_mean": 0.0008206626165095562, "completions/clipped_ratio": 0.38333335518836975, "completions/max_length": 2048.0, "completions/max_terminated_length": 856.0, "completions/mean_length": 1005.3333740234375, "completions/mean_terminated_length": 357.189208984375, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 8.980292320251465, "epoch": 0.5743440233236151, "frac_reward_zero_std": 0.0, "grad_norm": 0.0492168553173542, "learning_rate": 2e-05, "loss": 0.23936019837856293, "num_tokens": 28690597.0, "reward": -0.19088542461395264, "reward_std": 0.7735458612442017, "rewards/correctness/mean": 0.30000001192092896, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.4908854067325592, "rewards/length_penalty/std": 0.40846267342567444, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9593268036842346, "sampling/importance_sampling_ratio/min": 0.00016484098159708083, "sampling/sampling_logp_difference/max": 8.710529327392578, "sampling/sampling_logp_difference/mean": 0.04873416945338249, "step": 394, "step_time": 27.93874346697703 }, { "clip_ratio/high_max": 0.0006880600327955714, "clip_ratio/high_mean": 0.0006880600327955714, "clip_ratio/low_mean": 0.000325507739034947, "clip_ratio/low_min": 0.000325507739034947, "clip_ratio/region_mean": 0.0010135677536406245, "completions/clipped_ratio": 0.3500000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 907.0, "completions/mean_length": 963.2000732421875, "completions/mean_terminated_length": 379.0769348144531, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "entropy": 8.52564279238383, "epoch": 0.575801749271137, "frac_reward_zero_std": 0.0, "grad_norm": 0.04101765155792236, "learning_rate": 2e-05, "loss": 0.1389997899532318, "num_tokens": 28753479.0, "reward": -0.22031250596046448, "reward_std": 0.7098496556282043, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.4703125059604645, "rewards/length_penalty/std": 0.39678525924682617, "sampling/importance_sampling_ratio/max": 2.699448585510254, "sampling/importance_sampling_ratio/mean": 0.9617882370948792, "sampling/importance_sampling_ratio/min": 0.027962395921349525, "sampling/sampling_logp_difference/max": 3.576894760131836, "sampling/sampling_logp_difference/mean": 0.047623686492443085, "step": 395, "step_time": 27.296601990005 }, { "clip_ratio/high_max": 0.0005070790333168892, "clip_ratio/high_mean": 0.0005070790333168892, "clip_ratio/low_mean": 0.00023696535572526045, "clip_ratio/low_min": 0.00023696535572526045, "clip_ratio/region_mean": 0.0007440443951054476, "completions/clipped_ratio": 0.31666669249534607, "completions/max_length": 2048.0, "completions/max_terminated_length": 624.0, "completions/mean_length": 898.7333984375, "completions/mean_terminated_length": 366.1463317871094, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 8.171338319778442, "epoch": 0.577259475218659, "frac_reward_zero_std": 0.0, "grad_norm": 0.04592467471957207, "learning_rate": 2e-05, "loss": 0.21847057342529297, "num_tokens": 28812973.0, "reward": 0.01116536557674408, "reward_std": 0.7897411584854126, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.43883463740348816, "rewards/length_penalty/std": 0.38807380199432373, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9617645144462585, "sampling/importance_sampling_ratio/min": 0.024881262332201004, "sampling/sampling_logp_difference/max": 3.6936402320861816, "sampling/sampling_logp_difference/mean": 0.04758559912443161, "step": 396, "step_time": 26.56982400943525 }, { "clip_ratio/high_max": 0.0009843829563275601, "clip_ratio/high_mean": 0.0009843829563275601, "clip_ratio/low_mean": 0.00016447636517114006, "clip_ratio/low_min": 0.00016447636517114006, "clip_ratio/region_mean": 0.0011488593202860404, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 855.7667236328125, "completions/mean_terminated_length": 422.227294921875, "completions/min_length": 178.0, "completions/min_terminated_length": 178.0, "entropy": 7.143376429875691, "epoch": 0.5787172011661808, "frac_reward_zero_std": 0.0, "grad_norm": 0.04649930074810982, "learning_rate": 2e-05, "loss": 0.20125705003738403, "num_tokens": 28868649.0, "reward": 0.06547851860523224, "reward_std": 0.7758048176765442, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.41785481572151184, "rewards/length_penalty/std": 0.35812580585479736, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9663699269294739, "sampling/importance_sampling_ratio/min": 0.023360813036561012, "sampling/sampling_logp_difference/max": 3.75669527053833, "sampling/sampling_logp_difference/mean": 0.044312261044979095, "step": 397, "step_time": 26.61660780874081 }, { "clip_ratio/high_max": 0.0024000517975461357, "clip_ratio/high_mean": 0.0024000517975461357, "clip_ratio/low_mean": 0.00026665957799802226, "clip_ratio/low_min": 0.00026665957799802226, "clip_ratio/region_mean": 0.0026667113561416045, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 515.0, "completions/mean_length": 562.683349609375, "completions/mean_terminated_length": 334.173095703125, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 5.016673445701599, "epoch": 0.5801749271137027, "frac_reward_zero_std": 0.0, "grad_norm": 0.050205908715724945, "learning_rate": 2e-05, "loss": 0.17944487929344177, "num_tokens": 28906910.0, "reward": 0.3585856258869171, "reward_std": 0.6756696701049805, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.27474772930145264, "rewards/length_penalty/std": 0.2891711890697479, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9713447690010071, "sampling/importance_sampling_ratio/min": 0.002596322912722826, "sampling/sampling_logp_difference/max": 5.9536590576171875, "sampling/sampling_logp_difference/mean": 0.040170278400182724, "step": 398, "step_time": 24.927727278787643 }, { "clip_ratio/high_max": 0.0013288800206889089, "clip_ratio/high_mean": 0.0013288800206889089, "clip_ratio/low_mean": 0.0001266632607439533, "clip_ratio/low_min": 0.0001266632607439533, "clip_ratio/region_mean": 0.001455543305686054, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 915.0, "completions/mean_length": 851.36669921875, "completions/mean_terminated_length": 416.227294921875, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 7.008573532104492, "epoch": 0.5816326530612245, "frac_reward_zero_std": 0.0, "grad_norm": 0.04747510328888893, "learning_rate": 2e-05, "loss": 0.22528056800365448, "num_tokens": 28962872.0, "reward": 0.08429362624883652, "reward_std": 0.8125839829444885, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.4157063663005829, "rewards/length_penalty/std": 0.36536988615989685, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9659606218338013, "sampling/importance_sampling_ratio/min": 0.009075718931853771, "sampling/sampling_logp_difference/max": 4.702152729034424, "sampling/sampling_logp_difference/mean": 0.043395377695560455, "step": 399, "step_time": 26.27963155019097 }, { "clip_ratio/high_max": 0.000595281666998441, "clip_ratio/high_mean": 0.000595281666998441, "clip_ratio/low_mean": 0.0004387141962070018, "clip_ratio/low_min": 0.0004387141962070018, "clip_ratio/region_mean": 0.0010339958826079965, "completions/clipped_ratio": 0.28333336114883423, "completions/max_length": 2048.0, "completions/max_terminated_length": 1106.0, "completions/mean_length": 929.5000610351562, "completions/mean_terminated_length": 487.3023376464844, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "entropy": 7.1155978838602705, "epoch": 0.5830903790087464, "frac_reward_zero_std": 0.0, "grad_norm": 0.04315875098109245, "learning_rate": 2e-05, "loss": 0.1897212266921997, "num_tokens": 29025352.0, "reward": -0.0038574221543967724, "reward_std": 0.7555517554283142, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.453857421875, "rewards/length_penalty/std": 0.36007291078567505, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9673987030982971, "sampling/importance_sampling_ratio/min": 0.043289922177791595, "sampling/sampling_logp_difference/max": 3.1398353576660156, "sampling/sampling_logp_difference/mean": 0.0410989485681057, "step": 400, "step_time": 28.459511306136847 } ], "logging_steps": 1, "max_steps": 1200, "num_input_tokens_seen": 29025352, "num_train_epochs": 2, "save_steps": 200, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 10, "trial_name": null, "trial_params": null }