{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.2915451895043732, "eval_steps": 500, "global_step": 200, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1779.0, "completions/mean_length": 911.7000732421875, "completions/mean_terminated_length": 851.8947143554688, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.287322332461675, "epoch": 0.0014577259475218659, "frac_reward_zero_std": 0.0, "grad_norm": 0.009315529838204384, "learning_rate": 0.0, "loss": -0.00722887646406889, "num_tokens": 59102.0, "reward": 0.02150065265595913, "reward_std": 0.6429958939552307, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.4451660215854645, "rewards/length_penalty/std": 0.22199024260044098, "sampling/importance_sampling_ratio/max": 1.4242706298828125, "sampling/importance_sampling_ratio/mean": 0.9905919432640076, "sampling/importance_sampling_ratio/min": 0.39864858984947205, "sampling/sampling_logp_difference/max": 0.9196749925613403, "sampling/sampling_logp_difference/mean": 0.017188014462590218, "step": 1, "step_time": 27.266703790985048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1218.0, "completions/mean_length": 583.9833374023438, "completions/mean_terminated_length": 559.1694946289062, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "entropy": 0.18992788344621658, "epoch": 0.0029154518950437317, "frac_reward_zero_std": 0.0, "grad_norm": 0.01012346614152193, "learning_rate": 3.3333333333333335e-07, "loss": 0.0682297945022583, "num_tokens": 97601.0, "reward": 0.39818525314331055, "reward_std": 0.5346248745918274, "rewards/correctness/mean": 0.6833333373069763, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.28514811396598816, "rewards/length_penalty/std": 0.16064155101776123, "sampling/importance_sampling_ratio/max": 1.4211019277572632, "sampling/importance_sampling_ratio/mean": 0.9934695363044739, "sampling/importance_sampling_ratio/min": 0.6183077096939087, "sampling/sampling_logp_difference/max": 0.4807690382003784, "sampling/sampling_logp_difference/mean": 0.011985952034592628, "step": 2, "step_time": 24.735876325285062 }, { "clip_ratio/high_max": 0.0005826703830583332, "clip_ratio/high_mean": 0.0005826703830583332, "clip_ratio/low_mean": 0.00020714851901478445, "clip_ratio/low_min": 0.00020714851901478445, "clip_ratio/region_mean": 0.000789818882670564, "completions/clipped_ratio": 0.18333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 1824.0, "completions/mean_length": 1067.4000244140625, "completions/mean_terminated_length": 847.2652587890625, "completions/min_length": 468.0, "completions/min_terminated_length": 468.0, "entropy": 0.3251006404558818, "epoch": 0.004373177842565598, "frac_reward_zero_std": 0.0, "grad_norm": 0.015216746367514133, "learning_rate": 6.666666666666667e-07, "loss": 0.07915706932544708, "num_tokens": 167245.0, "reward": -0.0545247420668602, "reward_std": 0.721366822719574, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.521191418170929, "rewards/length_penalty/std": 0.289736270904541, "sampling/importance_sampling_ratio/max": 1.4119378328323364, "sampling/importance_sampling_ratio/mean": 0.9888848662376404, "sampling/importance_sampling_ratio/min": 0.5514586567878723, "sampling/sampling_logp_difference/max": 0.5951883792877197, "sampling/sampling_logp_difference/mean": 0.018833020702004433, "step": 3, "step_time": 28.364217409398407 }, { "clip_ratio/high_max": 0.0009626642034466689, "clip_ratio/high_mean": 0.0009626642034466689, "clip_ratio/low_mean": 8.59517022036016e-05, "clip_ratio/low_min": 8.59517022036016e-05, "clip_ratio/region_mean": 0.0010486159202021856, "completions/clipped_ratio": 0.0, "completions/max_length": 1864.0, "completions/max_terminated_length": 1864.0, "completions/mean_length": 536.9000244140625, "completions/mean_terminated_length": 536.9000244140625, "completions/min_length": 59.0, "completions/min_terminated_length": 59.0, "entropy": 0.2079171563188235, "epoch": 0.0058309037900874635, "frac_reward_zero_std": 0.0, "grad_norm": 0.009288053959608078, "learning_rate": 1.0000000000000002e-06, "loss": 0.0008611474186182022, "num_tokens": 202649.0, "reward": 0.32117515802383423, "reward_std": 0.5547496676445007, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.26215821504592896, "rewards/length_penalty/std": 0.22235317528247833, "sampling/importance_sampling_ratio/max": 1.4492052793502808, "sampling/importance_sampling_ratio/mean": 0.9927003383636475, "sampling/importance_sampling_ratio/min": 0.673969566822052, "sampling/sampling_logp_difference/max": 0.39457035064697266, "sampling/sampling_logp_difference/mean": 0.013110565021634102, "step": 4, "step_time": 22.774585011880845 }, { "clip_ratio/high_max": 0.0007607487593001375, "clip_ratio/high_mean": 0.0007607487593001375, "clip_ratio/low_mean": 0.00016298730891624777, "clip_ratio/low_min": 0.00016298730891624777, "clip_ratio/region_mean": 0.000923736069429045, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1981.0, "completions/mean_length": 957.6000366210938, "completions/mean_terminated_length": 836.4444580078125, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.2149273157119751, "epoch": 0.007288629737609329, "frac_reward_zero_std": 0.0, "grad_norm": 0.012508715502917767, "learning_rate": 1.3333333333333334e-06, "loss": 0.05253123119473457, "num_tokens": 265495.0, "reward": 0.16575521230697632, "reward_std": 0.6683671474456787, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.46757811307907104, "rewards/length_penalty/std": 0.2820192873477936, "sampling/importance_sampling_ratio/max": 1.648134708404541, "sampling/importance_sampling_ratio/mean": 0.9923359155654907, "sampling/importance_sampling_ratio/min": 0.6768563985824585, "sampling/sampling_logp_difference/max": 0.4996441602706909, "sampling/sampling_logp_difference/mean": 0.013547893613576889, "step": 5, "step_time": 28.106884445995092 }, { "clip_ratio/high_max": 0.0007083940787803537, "clip_ratio/high_mean": 0.0007083940787803537, "clip_ratio/low_mean": 0.00020570932732274136, "clip_ratio/low_min": 0.00020570932732274136, "clip_ratio/region_mean": 0.0009141034194423506, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1958.0, "completions/mean_length": 969.86669921875, "completions/mean_terminated_length": 754.239990234375, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.2867318441470464, "epoch": 0.008746355685131196, "frac_reward_zero_std": 0.0, "grad_norm": 0.012962858192622662, "learning_rate": 1.6666666666666667e-06, "loss": 0.032503724098205566, "num_tokens": 328167.0, "reward": 0.12643229961395264, "reward_std": 0.7710843086242676, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.4735676944255829, "rewards/length_penalty/std": 0.31948766112327576, "sampling/importance_sampling_ratio/max": 1.4769279956817627, "sampling/importance_sampling_ratio/mean": 0.9899801015853882, "sampling/importance_sampling_ratio/min": 0.5725703835487366, "sampling/sampling_logp_difference/max": 0.557619571685791, "sampling/sampling_logp_difference/mean": 0.01695270836353302, "step": 6, "step_time": 27.56186681916006 }, { "clip_ratio/high_max": 0.0011398453304233651, "clip_ratio/high_mean": 0.0011398453304233651, "clip_ratio/low_mean": 6.085470522521064e-05, "clip_ratio/low_min": 6.085470522521064e-05, "clip_ratio/region_mean": 0.0012007000429245334, "completions/clipped_ratio": 0.0, "completions/max_length": 1098.0, "completions/max_terminated_length": 1098.0, "completions/mean_length": 542.3833618164062, "completions/mean_terminated_length": 542.3833618164062, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.17402570943037668, "epoch": 0.01020408163265306, "frac_reward_zero_std": 0.0, "grad_norm": 0.009222477674484253, "learning_rate": 2.0000000000000003e-06, "loss": 0.013532082550227642, "num_tokens": 365240.0, "reward": 0.6518310904502869, "reward_std": 0.3150613605976105, "rewards/correctness/mean": 0.9166666865348816, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.2648356258869171, "rewards/length_penalty/std": 0.0952993780374527, "sampling/importance_sampling_ratio/max": 1.411893367767334, "sampling/importance_sampling_ratio/mean": 0.9938415288925171, "sampling/importance_sampling_ratio/min": 0.6583556532859802, "sampling/sampling_logp_difference/max": 0.41800999641418457, "sampling/sampling_logp_difference/mean": 0.011639133095741272, "step": 7, "step_time": 14.554296029964462 }, { "clip_ratio/high_max": 0.0007130262577751031, "clip_ratio/high_mean": 0.0007130262577751031, "clip_ratio/low_mean": 9.414875239599496e-05, "clip_ratio/low_min": 9.414875239599496e-05, "clip_ratio/region_mean": 0.0008071750247230133, "completions/clipped_ratio": 0.0, "completions/max_length": 1923.0, "completions/max_terminated_length": 1923.0, "completions/mean_length": 596.7000122070312, "completions/mean_terminated_length": 596.7000122070312, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.23006930450598398, "epoch": 0.011661807580174927, "frac_reward_zero_std": 0.0, "grad_norm": 0.009591356851160526, "learning_rate": 2.3333333333333336e-06, "loss": 0.07374510169029236, "num_tokens": 407292.0, "reward": 0.7086426019668579, "reward_std": 0.19280359148979187, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.2913574278354645, "rewards/length_penalty/std": 0.19280359148979187, "sampling/importance_sampling_ratio/max": 1.450048565864563, "sampling/importance_sampling_ratio/mean": 0.991736888885498, "sampling/importance_sampling_ratio/min": 0.6695584058761597, "sampling/sampling_logp_difference/max": 0.4011368751525879, "sampling/sampling_logp_difference/mean": 0.014551110565662384, "step": 8, "step_time": 25.349019592627883 }, { "clip_ratio/high_max": 0.0003552542487644435, "clip_ratio/high_mean": 0.0003552542487644435, "clip_ratio/low_mean": 0.0001425145031438054, "clip_ratio/low_min": 0.0001425145031438054, "clip_ratio/region_mean": 0.0004977687591842065, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1988.0, "completions/mean_length": 907.6834106445312, "completions/mean_terminated_length": 804.0181274414062, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.23283551881710687, "epoch": 0.013119533527696793, "frac_reward_zero_std": 0.0, "grad_norm": 0.011995715089142323, "learning_rate": 2.666666666666667e-06, "loss": -0.005120586603879929, "num_tokens": 466993.0, "reward": 0.04012858122587204, "reward_std": 0.6957505345344543, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.44320476055145264, "rewards/length_penalty/std": 0.28446123003959656, "sampling/importance_sampling_ratio/max": 1.4668807983398438, "sampling/importance_sampling_ratio/mean": 0.9919294118881226, "sampling/importance_sampling_ratio/min": 0.6455317139625549, "sampling/sampling_logp_difference/max": 0.4376809597015381, "sampling/sampling_logp_difference/mean": 0.014371889643371105, "step": 9, "step_time": 27.68978225090541 }, { "clip_ratio/high_max": 0.0007404128070144603, "clip_ratio/high_mean": 0.0007404128070144603, "clip_ratio/low_mean": 0.00014422326057683676, "clip_ratio/low_min": 0.00014422326057683676, "clip_ratio/region_mean": 0.0008846360724419355, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 2032.0, "completions/mean_length": 1016.4500732421875, "completions/mean_terminated_length": 810.1399536132812, "completions/min_length": 385.0, "completions/min_terminated_length": 385.0, "entropy": 0.2652890533208847, "epoch": 0.014577259475218658, "frac_reward_zero_std": 0.0, "grad_norm": 0.01154837105423212, "learning_rate": 3e-06, "loss": 0.05848747491836548, "num_tokens": 534050.0, "reward": 0.17035320401191711, "reward_std": 0.6838861703872681, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.4963134825229645, "rewards/length_penalty/std": 0.26304200291633606, "sampling/importance_sampling_ratio/max": 1.4769505262374878, "sampling/importance_sampling_ratio/mean": 0.9908170104026794, "sampling/importance_sampling_ratio/min": 0.6580171585083008, "sampling/sampling_logp_difference/max": 0.41852426528930664, "sampling/sampling_logp_difference/mean": 0.016183653846383095, "step": 10, "step_time": 28.15277793817222 }, { "clip_ratio/high_max": 0.001398060965584591, "clip_ratio/high_mean": 0.001398060965584591, "clip_ratio/low_mean": 0.0001395033480851756, "clip_ratio/low_min": 0.0001395033480851756, "clip_ratio/region_mean": 0.0015375642785026382, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1751.0, "completions/mean_length": 1006.7500610351562, "completions/mean_terminated_length": 869.2264404296875, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.3573797369996707, "epoch": 0.016034985422740525, "frac_reward_zero_std": 0.0, "grad_norm": 0.016619419679045677, "learning_rate": 3.3333333333333333e-06, "loss": 0.01292102038860321, "num_tokens": 599195.0, "reward": -0.30824384093284607, "reward_std": 0.4079125225543976, "rewards/correctness/mean": 0.18333333730697632, "rewards/correctness/std": 0.39020493626594543, "rewards/length_penalty/mean": -0.4915771484375, "rewards/length_penalty/std": 0.29589328169822693, "sampling/importance_sampling_ratio/max": 1.4458197355270386, "sampling/importance_sampling_ratio/mean": 0.9881013035774231, "sampling/importance_sampling_ratio/min": 0.5618796944618225, "sampling/sampling_logp_difference/max": 0.5764675140380859, "sampling/sampling_logp_difference/mean": 0.019838672131299973, "step": 11, "step_time": 28.012359508080408 }, { "clip_ratio/high_max": 0.0007473976923696076, "clip_ratio/high_mean": 0.0007473976923696076, "clip_ratio/low_mean": 0.0001535463670734316, "clip_ratio/low_min": 0.0001535463670734316, "clip_ratio/region_mean": 0.0009009440545924008, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1887.0, "completions/mean_length": 823.5833740234375, "completions/mean_terminated_length": 661.867919921875, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.28605597962935764, "epoch": 0.01749271137026239, "frac_reward_zero_std": 0.0, "grad_norm": 0.011755434796214104, "learning_rate": 3.6666666666666666e-06, "loss": 0.017311107367277145, "num_tokens": 654090.0, "reward": 0.19785970449447632, "reward_std": 0.6898223757743835, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.4021402895450592, "rewards/length_penalty/std": 0.2941838204860687, "sampling/importance_sampling_ratio/max": 1.4662060737609863, "sampling/importance_sampling_ratio/mean": 0.9898833632469177, "sampling/importance_sampling_ratio/min": 0.6318491697311401, "sampling/sampling_logp_difference/max": 0.4591045379638672, "sampling/sampling_logp_difference/mean": 0.01767781749367714, "step": 12, "step_time": 26.969347662990913 }, { "clip_ratio/high_max": 0.0014183916015705715, "clip_ratio/high_mean": 0.0014183916015705715, "clip_ratio/low_mean": 6.0735284932889044e-05, "clip_ratio/low_min": 6.0735284932889044e-05, "clip_ratio/region_mean": 0.0014791268719515454, "completions/clipped_ratio": 0.0, "completions/max_length": 1254.0, "completions/max_terminated_length": 1254.0, "completions/mean_length": 533.0333862304688, "completions/mean_terminated_length": 533.0333862304688, "completions/min_length": 73.0, "completions/min_terminated_length": 73.0, "entropy": 0.26387997219959897, "epoch": 0.018950437317784258, "frac_reward_zero_std": 0.0, "grad_norm": 0.00885060615837574, "learning_rate": 4.000000000000001e-06, "loss": 0.009860651567578316, "num_tokens": 690932.0, "reward": 0.3063965141773224, "reward_std": 0.5047494769096375, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971747398376465, "rewards/length_penalty/mean": -0.2602701783180237, "rewards/length_penalty/std": 0.14477114379405975, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9907361268997192, "sampling/importance_sampling_ratio/min": 0.580272376537323, "sampling/sampling_logp_difference/max": 1.1491729021072388, "sampling/sampling_logp_difference/mean": 0.01654963009059429, "step": 13, "step_time": 16.872128481976688 }, { "clip_ratio/high_max": 0.0005438411681097932, "clip_ratio/high_mean": 0.0005438411681097932, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0005438411681097932, "completions/clipped_ratio": 0.0, "completions/max_length": 1314.0, "completions/max_terminated_length": 1314.0, "completions/mean_length": 563.433349609375, "completions/mean_terminated_length": 563.433349609375, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.16707361737887064, "epoch": 0.02040816326530612, "frac_reward_zero_std": 0.0, "grad_norm": 0.0070292409509420395, "learning_rate": 4.333333333333334e-06, "loss": 0.01137271523475647, "num_tokens": 728548.0, "reward": 0.3248860836029053, "reward_std": 0.4802418351173401, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.27511394023895264, "rewards/length_penalty/std": 0.10393763333559036, "sampling/importance_sampling_ratio/max": 1.4095503091812134, "sampling/importance_sampling_ratio/mean": 0.9941017031669617, "sampling/importance_sampling_ratio/min": 0.6589340567588806, "sampling/sampling_logp_difference/max": 0.4171319007873535, "sampling/sampling_logp_difference/mean": 0.010680951178073883, "step": 14, "step_time": 16.798709742026404 }, { "clip_ratio/high_max": 0.0009462485904805362, "clip_ratio/high_mean": 0.0009462485904805362, "clip_ratio/low_mean": 0.0001415904177216968, "clip_ratio/low_min": 0.0001415904177216968, "clip_ratio/region_mean": 0.001087839011840212, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 2025.0, "completions/mean_length": 1070.9500732421875, "completions/mean_terminated_length": 875.5399780273438, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.23209517200787863, "epoch": 0.021865889212827987, "frac_reward_zero_std": 0.0, "grad_norm": 0.010992632247507572, "learning_rate": 4.666666666666667e-06, "loss": 0.004310145974159241, "num_tokens": 796855.0, "reward": 0.16040854156017303, "reward_std": 0.6544270515441895, "rewards/correctness/mean": 0.6833333373069763, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.5229247808456421, "rewards/length_penalty/std": 0.26274341344833374, "sampling/importance_sampling_ratio/max": 1.4131360054016113, "sampling/importance_sampling_ratio/mean": 0.9919527769088745, "sampling/importance_sampling_ratio/min": 0.5095058083534241, "sampling/sampling_logp_difference/max": 0.674314022064209, "sampling/sampling_logp_difference/mean": 0.013961256481707096, "step": 15, "step_time": 27.5681957276538 }, { "clip_ratio/high_max": 0.0010235675047927846, "clip_ratio/high_mean": 0.0010235675047927846, "clip_ratio/low_mean": 4.2773429110335805e-05, "clip_ratio/low_min": 4.2773429110335805e-05, "clip_ratio/region_mean": 0.0010663409387537588, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1847.0, "completions/mean_length": 726.7333984375, "completions/mean_terminated_length": 681.1724243164062, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.279613917072614, "epoch": 0.023323615160349854, "frac_reward_zero_std": 0.0, "grad_norm": 0.015736393630504608, "learning_rate": 5e-06, "loss": 0.04625968262553215, "num_tokens": 845189.0, "reward": 0.16181641817092896, "reward_std": 0.5572150945663452, "rewards/correctness/mean": 0.5166666507720947, "rewards/correctness/std": 0.5039393305778503, "rewards/length_penalty/mean": -0.35485026240348816, "rewards/length_penalty/std": 0.18019931018352509, "sampling/importance_sampling_ratio/max": 1.424391269683838, "sampling/importance_sampling_ratio/mean": 0.9905226826667786, "sampling/importance_sampling_ratio/min": 0.676193118095398, "sampling/sampling_logp_difference/max": 0.39127659797668457, "sampling/sampling_logp_difference/mean": 0.016356047242879868, "step": 16, "step_time": 26.450964051065966 }, { "clip_ratio/high_max": 0.001121756814730664, "clip_ratio/high_mean": 0.001121756814730664, "clip_ratio/low_mean": 2.9556067602243274e-05, "clip_ratio/low_min": 2.9556067602243274e-05, "clip_ratio/region_mean": 0.001151312889608865, "completions/clipped_ratio": 0.0, "completions/max_length": 1359.0, "completions/max_terminated_length": 1359.0, "completions/mean_length": 479.35003662109375, "completions/mean_terminated_length": 479.35003662109375, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.23176098863283792, "epoch": 0.02478134110787172, "frac_reward_zero_std": 0.0, "grad_norm": 0.008707236498594284, "learning_rate": 5.333333333333334e-06, "loss": -0.023332105949521065, "num_tokens": 883500.0, "reward": 0.515942394733429, "reward_std": 0.46456581354141235, "rewards/correctness/mean": 0.75, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.23405762016773224, "rewards/length_penalty/std": 0.12105229496955872, "sampling/importance_sampling_ratio/max": 1.437240719795227, "sampling/importance_sampling_ratio/mean": 0.991898775100708, "sampling/importance_sampling_ratio/min": 0.6554322242736816, "sampling/sampling_logp_difference/max": 0.42246031761169434, "sampling/sampling_logp_difference/mean": 0.01451260969042778, "step": 17, "step_time": 18.0038778539747 }, { "clip_ratio/high_max": 0.0006659166829194874, "clip_ratio/high_mean": 0.0006659166829194874, "clip_ratio/low_mean": 0.00017530394082617326, "clip_ratio/low_min": 0.00017530394082617326, "clip_ratio/region_mean": 0.0008412206370849162, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 2021.0, "completions/mean_length": 1156.3333740234375, "completions/mean_terminated_length": 1038.5660400390625, "completions/min_length": 392.0, "completions/min_terminated_length": 392.0, "entropy": 0.2782107988993327, "epoch": 0.026239067055393587, "frac_reward_zero_std": 0.0, "grad_norm": 0.01186887826770544, "learning_rate": 5.666666666666667e-06, "loss": 0.0311305969953537, "num_tokens": 957550.0, "reward": -0.4146158993244171, "reward_std": 0.46414339542388916, "rewards/correctness/mean": 0.15000000596046448, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.5646159052848816, "rewards/length_penalty/std": 0.24389268457889557, "sampling/importance_sampling_ratio/max": 1.4626619815826416, "sampling/importance_sampling_ratio/mean": 0.9902209639549255, "sampling/importance_sampling_ratio/min": 0.6455824971199036, "sampling/sampling_logp_difference/max": 0.43760228157043457, "sampling/sampling_logp_difference/mean": 0.01654873974621296, "step": 18, "step_time": 28.444838002789766 }, { "clip_ratio/high_max": 0.000992356882003757, "clip_ratio/high_mean": 0.000992356882003757, "clip_ratio/low_mean": 3.6043830429359026e-05, "clip_ratio/low_min": 3.6043830429359026e-05, "clip_ratio/region_mean": 0.001028400714858435, "completions/clipped_ratio": 0.0, "completions/max_length": 1005.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 410.4000244140625, "completions/mean_terminated_length": 410.4000244140625, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "entropy": 0.22018084675073624, "epoch": 0.027696793002915453, "frac_reward_zero_std": 0.0, "grad_norm": 0.009298046119511127, "learning_rate": 6e-06, "loss": -0.0038212621584534645, "num_tokens": 985114.0, "reward": 0.5329427123069763, "reward_std": 0.43304309248924255, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.20039062201976776, "rewards/length_penalty/std": 0.09832219034433365, "sampling/importance_sampling_ratio/max": 1.4597746133804321, "sampling/importance_sampling_ratio/mean": 0.9921642541885376, "sampling/importance_sampling_ratio/min": 0.6524658799171448, "sampling/sampling_logp_difference/max": 0.42699646949768066, "sampling/sampling_logp_difference/mean": 0.014085552655160427, "step": 19, "step_time": 13.759585740976036 }, { "clip_ratio/high_max": 0.0007611963568100085, "clip_ratio/high_mean": 0.0007611963568100085, "clip_ratio/low_mean": 0.00018029654286995841, "clip_ratio/low_min": 0.00018029654286995841, "clip_ratio/region_mean": 0.0009414929130192226, "completions/clipped_ratio": 0.18333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 2047.0, "completions/mean_length": 944.2500610351562, "completions/mean_terminated_length": 696.4693603515625, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.2984036107858022, "epoch": 0.029154518950437316, "frac_reward_zero_std": 0.0, "grad_norm": 0.014993884600698948, "learning_rate": 6.333333333333333e-06, "loss": 0.0036488808691501617, "num_tokens": 1045869.0, "reward": -0.1277262419462204, "reward_std": 0.6400496959686279, "rewards/correctness/mean": 0.3333333432674408, "rewards/correctness/std": 0.4753826856613159, "rewards/length_penalty/mean": -0.4610595703125, "rewards/length_penalty/std": 0.329764187335968, "sampling/importance_sampling_ratio/max": 1.7107930183410645, "sampling/importance_sampling_ratio/mean": 0.9897626638412476, "sampling/importance_sampling_ratio/min": 0.5234970450401306, "sampling/sampling_logp_difference/max": 0.647223949432373, "sampling/sampling_logp_difference/mean": 0.017825914546847343, "step": 20, "step_time": 27.421940207248554 }, { "clip_ratio/high_max": 0.0008657800984413674, "clip_ratio/high_mean": 0.0008657800984413674, "clip_ratio/low_mean": 0.00020750967329756045, "clip_ratio/low_min": 0.00020750967329756045, "clip_ratio/region_mean": 0.0010732897547616933, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1960.0, "completions/mean_length": 720.183349609375, "completions/mean_terminated_length": 697.677978515625, "completions/min_length": 164.0, "completions/min_terminated_length": 164.0, "entropy": 0.2126310889919599, "epoch": 0.030612244897959183, "frac_reward_zero_std": 0.0, "grad_norm": 0.011873041279613972, "learning_rate": 6.666666666666667e-06, "loss": 0.022917810827493668, "num_tokens": 1097530.0, "reward": 0.31501466035842896, "reward_std": 0.6336944699287415, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.35165202617645264, "rewards/length_penalty/std": 0.21549133956432343, "sampling/importance_sampling_ratio/max": 1.4036577939987183, "sampling/importance_sampling_ratio/mean": 0.9929825067520142, "sampling/importance_sampling_ratio/min": 0.641616702079773, "sampling/sampling_logp_difference/max": 0.44376420974731445, "sampling/sampling_logp_difference/mean": 0.01318247988820076, "step": 21, "step_time": 28.00862360955216 }, { "clip_ratio/high_max": 0.0009326817137965312, "clip_ratio/high_mean": 0.0009326817137965312, "clip_ratio/low_mean": 2.0387359351540606e-05, "clip_ratio/low_min": 2.0387359351540606e-05, "clip_ratio/region_mean": 0.0009530690828493485, "completions/clipped_ratio": 0.0, "completions/max_length": 1656.0, "completions/max_terminated_length": 1656.0, "completions/mean_length": 784.3833618164062, "completions/mean_terminated_length": 784.3833618164062, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.21914158513148627, "epoch": 0.03206997084548105, "frac_reward_zero_std": 0.0, "grad_norm": 0.01254537794739008, "learning_rate": 7e-06, "loss": -0.0010693036019802094, "num_tokens": 1150133.0, "reward": 0.5503336787223816, "reward_std": 0.34320735931396484, "rewards/correctness/mean": 0.9333333373069763, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.3829996883869171, "rewards/length_penalty/std": 0.18697305023670197, "sampling/importance_sampling_ratio/max": 1.4669926166534424, "sampling/importance_sampling_ratio/mean": 0.9924184679985046, "sampling/importance_sampling_ratio/min": 0.6394065618515015, "sampling/sampling_logp_difference/max": 0.44721484184265137, "sampling/sampling_logp_difference/mean": 0.01387734618037939, "step": 22, "step_time": 22.649415029911324 }, { "clip_ratio/high_max": 0.000687097145904166, "clip_ratio/high_mean": 0.000687097145904166, "clip_ratio/low_mean": 0.00010386897944651234, "clip_ratio/low_min": 0.00010386897944651234, "clip_ratio/region_mean": 0.0007909661168620611, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1801.0, "completions/mean_length": 942.7167358398438, "completions/mean_terminated_length": 884.5438842773438, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.20660034815470377, "epoch": 0.033527696793002916, "frac_reward_zero_std": 0.0, "grad_norm": 0.014352154918015003, "learning_rate": 7.333333333333333e-06, "loss": 0.01802755892276764, "num_tokens": 1211676.0, "reward": -0.21031087636947632, "reward_std": 0.5049320459365845, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.4603108763694763, "rewards/length_penalty/std": 0.2326008826494217, "sampling/importance_sampling_ratio/max": 1.4130291938781738, "sampling/importance_sampling_ratio/mean": 0.9927947521209717, "sampling/importance_sampling_ratio/min": 0.4475542902946472, "sampling/sampling_logp_difference/max": 0.803957462310791, "sampling/sampling_logp_difference/mean": 0.012972496449947357, "step": 23, "step_time": 27.452100809896365 }, { "clip_ratio/high_max": 0.00119293418538291, "clip_ratio/high_mean": 0.00119293418538291, "clip_ratio/low_mean": 0.00015429374737626253, "clip_ratio/low_min": 0.00015429374737626253, "clip_ratio/region_mean": 0.0013472279048680018, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1954.0, "completions/mean_length": 1060.75, "completions/mean_terminated_length": 813.9375, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.3705875674883525, "epoch": 0.03498542274052478, "frac_reward_zero_std": 0.0, "grad_norm": 0.013515997678041458, "learning_rate": 7.666666666666667e-06, "loss": 0.03517736494541168, "num_tokens": 1281111.0, "reward": -0.001277669332921505, "reward_std": 0.719670832157135, "rewards/correctness/mean": 0.5166666507720947, "rewards/correctness/std": 0.5039393305778503, "rewards/length_penalty/mean": -0.5179443359375, "rewards/length_penalty/std": 0.3272433280944824, "sampling/importance_sampling_ratio/max": 1.4604820013046265, "sampling/importance_sampling_ratio/mean": 0.9876757860183716, "sampling/importance_sampling_ratio/min": 0.5807204246520996, "sampling/sampling_logp_difference/max": 0.5434858798980713, "sampling/sampling_logp_difference/mean": 0.020977798849344254, "step": 24, "step_time": 27.973477198043838 }, { "clip_ratio/high_max": 0.0006930709569132887, "clip_ratio/high_mean": 0.0006930709569132887, "clip_ratio/low_mean": 6.285587854411763e-05, "clip_ratio/low_min": 6.285587854411763e-05, "clip_ratio/region_mean": 0.0007559268318194275, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1684.0, "completions/mean_length": 774.4833984375, "completions/mean_terminated_length": 752.8983154296875, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "entropy": 0.18978050351142883, "epoch": 0.03644314868804665, "frac_reward_zero_std": 0.0, "grad_norm": 0.010804434306919575, "learning_rate": 8.000000000000001e-06, "loss": 0.03464656323194504, "num_tokens": 1331990.0, "reward": 0.3385010063648224, "reward_std": 0.5436288714408875, "rewards/correctness/mean": 0.7166666388511658, "rewards/correctness/std": 0.4544196128845215, "rewards/length_penalty/mean": -0.37816569209098816, "rewards/length_penalty/std": 0.204934760928154, "sampling/importance_sampling_ratio/max": 1.4578324556350708, "sampling/importance_sampling_ratio/mean": 0.993187665939331, "sampling/importance_sampling_ratio/min": 0.6711965203285217, "sampling/sampling_logp_difference/max": 0.398693323135376, "sampling/sampling_logp_difference/mean": 0.012119485065340996, "step": 25, "step_time": 25.95817614090629 }, { "clip_ratio/high_max": 0.00038442869602780166, "clip_ratio/high_mean": 0.00038442869602780166, "clip_ratio/low_mean": 0.00017288892801540592, "clip_ratio/low_min": 0.00017288892801540592, "clip_ratio/region_mean": 0.0005573176216178884, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1915.0, "completions/mean_length": 670.9000244140625, "completions/mean_terminated_length": 545.7090454101562, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.16129851341247559, "epoch": 0.037900874635568516, "frac_reward_zero_std": 0.0, "grad_norm": 0.009562763385474682, "learning_rate": 8.333333333333334e-06, "loss": 0.017397675663232803, "num_tokens": 1376984.0, "reward": 0.33907878398895264, "reward_std": 0.6748707294464111, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.32758790254592896, "rewards/length_penalty/std": 0.2830147445201874, "sampling/importance_sampling_ratio/max": 1.4385851621627808, "sampling/importance_sampling_ratio/mean": 0.9943342208862305, "sampling/importance_sampling_ratio/min": 0.5278142690658569, "sampling/sampling_logp_difference/max": 0.6390109062194824, "sampling/sampling_logp_difference/mean": 0.010507638566195965, "step": 26, "step_time": 25.522193972254172 }, { "clip_ratio/high_max": 0.0009138718208608528, "clip_ratio/high_mean": 0.0009138718208608528, "clip_ratio/low_mean": 0.00015279000460092598, "clip_ratio/low_min": 0.00015279000460092598, "clip_ratio/region_mean": 0.0010666618375883747, "completions/clipped_ratio": 0.0, "completions/max_length": 2034.0, "completions/max_terminated_length": 2034.0, "completions/mean_length": 676.6333618164062, "completions/mean_terminated_length": 676.6333618164062, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.23284050822257996, "epoch": 0.03935860058309038, "frac_reward_zero_std": 0.0, "grad_norm": 0.012365774251520634, "learning_rate": 8.666666666666668e-06, "loss": 0.024310387670993805, "num_tokens": 1422522.0, "reward": 0.5196126699447632, "reward_std": 0.4647842049598694, "rewards/correctness/mean": 0.8500000238418579, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.3303873836994171, "rewards/length_penalty/std": 0.17094206809997559, "sampling/importance_sampling_ratio/max": 1.4408317804336548, "sampling/importance_sampling_ratio/mean": 0.9918356537818909, "sampling/importance_sampling_ratio/min": 0.5742970108985901, "sampling/sampling_logp_difference/max": 0.5546085834503174, "sampling/sampling_logp_difference/mean": 0.015102035365998745, "step": 27, "step_time": 24.69438692368567 }, { "clip_ratio/high_max": 0.0010074215630690257, "clip_ratio/high_mean": 0.0010074215630690257, "clip_ratio/low_mean": 6.152293644845486e-05, "clip_ratio/low_min": 6.152293644845486e-05, "clip_ratio/region_mean": 0.0010689445189200342, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1912.0, "completions/mean_length": 750.433349609375, "completions/mean_terminated_length": 705.6896362304688, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.29244403292735416, "epoch": 0.04081632653061224, "frac_reward_zero_std": 0.0, "grad_norm": 0.015244685113430023, "learning_rate": 9e-06, "loss": 0.05068972706794739, "num_tokens": 1471248.0, "reward": 0.30024415254592896, "reward_std": 0.6113126277923584, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.36642253398895264, "rewards/length_penalty/std": 0.24141043424606323, "sampling/importance_sampling_ratio/max": 1.5158343315124512, "sampling/importance_sampling_ratio/mean": 0.9897527694702148, "sampling/importance_sampling_ratio/min": 0.6490998268127441, "sampling/sampling_logp_difference/max": 0.43216872215270996, "sampling/sampling_logp_difference/mean": 0.0177592895925045, "step": 28, "step_time": 25.77531434223056 }, { "clip_ratio/high_max": 0.0013134582453252126, "clip_ratio/high_mean": 0.0013134582453252126, "clip_ratio/low_mean": 0.00016424481145804748, "clip_ratio/low_min": 0.00016424481145804748, "clip_ratio/region_mean": 0.0014777030640592177, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1819.0, "completions/mean_length": 803.2333984375, "completions/mean_terminated_length": 737.7192993164062, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.3203006610274315, "epoch": 0.04227405247813411, "frac_reward_zero_std": 0.0, "grad_norm": 0.012082278728485107, "learning_rate": 9.333333333333334e-06, "loss": 0.011175485327839851, "num_tokens": 1524632.0, "reward": 0.3244629204273224, "reward_std": 0.6027631163597107, "rewards/correctness/mean": 0.7166666388511658, "rewards/correctness/std": 0.4544196128845215, "rewards/length_penalty/mean": -0.39220377802848816, "rewards/length_penalty/std": 0.22215788066387177, "sampling/importance_sampling_ratio/max": 1.4445055723190308, "sampling/importance_sampling_ratio/mean": 0.9887529611587524, "sampling/importance_sampling_ratio/min": 0.6647764444351196, "sampling/sampling_logp_difference/max": 0.40830445289611816, "sampling/sampling_logp_difference/mean": 0.01917846128344536, "step": 29, "step_time": 25.925504898885265 }, { "clip_ratio/high_max": 0.0010785255290102214, "clip_ratio/high_mean": 0.0010785255290102214, "clip_ratio/low_mean": 0.00013830826355842873, "clip_ratio/low_min": 0.00013830826355842873, "clip_ratio/region_mean": 0.0012168337949939694, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1999.0, "completions/mean_length": 915.0000610351562, "completions/mean_terminated_length": 789.1111450195312, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.34353333214918774, "epoch": 0.043731778425655975, "frac_reward_zero_std": 0.0, "grad_norm": 0.014864777214825153, "learning_rate": 9.666666666666667e-06, "loss": 0.005697796121239662, "num_tokens": 1586022.0, "reward": 0.15322266519069672, "reward_std": 0.6906428933143616, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.44677734375, "rewards/length_penalty/std": 0.30850422382354736, "sampling/importance_sampling_ratio/max": 1.4533123970031738, "sampling/importance_sampling_ratio/mean": 0.9885200262069702, "sampling/importance_sampling_ratio/min": 0.547637939453125, "sampling/sampling_logp_difference/max": 0.6021409034729004, "sampling/sampling_logp_difference/mean": 0.01985924318432808, "step": 30, "step_time": 26.621315252035856 }, { "clip_ratio/high_max": 0.001238937091936047, "clip_ratio/high_mean": 0.001238937091936047, "clip_ratio/low_mean": 9.135761744497965e-05, "clip_ratio/low_min": 9.135761744497965e-05, "clip_ratio/region_mean": 0.001330294714231665, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1970.0, "completions/mean_length": 852.5833740234375, "completions/mean_terminated_length": 789.6666870117188, "completions/min_length": 401.0, "completions/min_terminated_length": 401.0, "entropy": 0.2833813379208247, "epoch": 0.04518950437317784, "frac_reward_zero_std": 0.0, "grad_norm": 0.01020822860300541, "learning_rate": 1e-05, "loss": -0.025425449013710022, "num_tokens": 1642917.0, "reward": 0.20036622881889343, "reward_std": 0.5892859697341919, "rewards/correctness/mean": 0.6166666746139526, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.4163004457950592, "rewards/length_penalty/std": 0.2239331156015396, "sampling/importance_sampling_ratio/max": 1.4509061574935913, "sampling/importance_sampling_ratio/mean": 0.9902245998382568, "sampling/importance_sampling_ratio/min": 0.6482113599777222, "sampling/sampling_logp_difference/max": 0.43353843688964844, "sampling/sampling_logp_difference/mean": 0.017317544668912888, "step": 31, "step_time": 26.81202861131169 }, { "clip_ratio/high_max": 0.000777290168722781, "clip_ratio/high_mean": 0.000777290168722781, "clip_ratio/low_mean": 0.00011783391770829137, "clip_ratio/low_min": 0.00011783391770829137, "clip_ratio/region_mean": 0.0008951240840057532, "completions/clipped_ratio": 0.0, "completions/max_length": 1870.0, "completions/max_terminated_length": 1870.0, "completions/mean_length": 869.4667358398438, "completions/mean_terminated_length": 869.4667358398438, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.23462951431671777, "epoch": 0.04664723032069971, "frac_reward_zero_std": 0.0, "grad_norm": 0.010549271479249, "learning_rate": 1.0333333333333335e-05, "loss": -0.00782666727900505, "num_tokens": 1700595.0, "reward": 0.19212239980697632, "reward_std": 0.6066915392875671, "rewards/correctness/mean": 0.6166666746139526, "rewards/correctness/std": 0.4903014302253723, "rewards/length_penalty/mean": -0.4245442748069763, "rewards/length_penalty/std": 0.17661738395690918, "sampling/importance_sampling_ratio/max": 1.4387601613998413, "sampling/importance_sampling_ratio/mean": 0.9920538067817688, "sampling/importance_sampling_ratio/min": 0.6378967761993408, "sampling/sampling_logp_difference/max": 0.44957876205444336, "sampling/sampling_logp_difference/mean": 0.014624694362282753, "step": 32, "step_time": 24.085791839752346 }, { "clip_ratio/high_max": 0.0010240920237265527, "clip_ratio/high_mean": 0.0010240920237265527, "clip_ratio/low_mean": 0.0001451966333358238, "clip_ratio/low_min": 0.0001451966333358238, "clip_ratio/region_mean": 0.0011692886667636533, "completions/clipped_ratio": 0.25, "completions/max_length": 2048.0, "completions/max_terminated_length": 1936.0, "completions/mean_length": 1480.666748046875, "completions/mean_terminated_length": 1291.5555419921875, "completions/min_length": 490.0, "completions/min_terminated_length": 490.0, "entropy": 0.3056724816560745, "epoch": 0.048104956268221574, "frac_reward_zero_std": 0.0, "grad_norm": 0.015298835933208466, "learning_rate": 1.0666666666666667e-05, "loss": 0.06860838085412979, "num_tokens": 1795955.0, "reward": -0.25631511211395264, "reward_std": 0.631487250328064, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.7229817509651184, "rewards/length_penalty/std": 0.22549602389335632, "sampling/importance_sampling_ratio/max": 1.5993895530700684, "sampling/importance_sampling_ratio/mean": 0.9892873167991638, "sampling/importance_sampling_ratio/min": 0.5515028238296509, "sampling/sampling_logp_difference/max": 0.5951082706451416, "sampling/sampling_logp_difference/mean": 0.018112879246473312, "step": 33, "step_time": 30.11426667473279 }, { "clip_ratio/high_max": 0.0005963442818028852, "clip_ratio/high_mean": 0.0005963442818028852, "clip_ratio/low_mean": 8.18812283493268e-05, "clip_ratio/low_min": 8.18812283493268e-05, "clip_ratio/region_mean": 0.0006782255028762544, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1275.0, "completions/mean_length": 730.4000244140625, "completions/mean_terminated_length": 584.0, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.2809029494722684, "epoch": 0.04956268221574344, "frac_reward_zero_std": 0.0, "grad_norm": 0.012408322654664516, "learning_rate": 1.1000000000000001e-05, "loss": 0.055800504982471466, "num_tokens": 1845059.0, "reward": 0.22669272124767303, "reward_std": 0.6471343636512756, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.35664063692092896, "rewards/length_penalty/std": 0.2492990493774414, "sampling/importance_sampling_ratio/max": 1.6343612670898438, "sampling/importance_sampling_ratio/mean": 0.9900498986244202, "sampling/importance_sampling_ratio/min": 0.6331878900527954, "sampling/sampling_logp_difference/max": 0.491252064704895, "sampling/sampling_logp_difference/mean": 0.017236072570085526, "step": 34, "step_time": 25.358212631428614 }, { "clip_ratio/high_max": 0.0007682585661920408, "clip_ratio/high_mean": 0.0007682585661920408, "clip_ratio/low_mean": 0.0001700489034798617, "clip_ratio/low_min": 0.0001700489034798617, "clip_ratio/region_mean": 0.0009383074842238178, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1947.0, "completions/mean_length": 904.5167236328125, "completions/mean_terminated_length": 777.4629516601562, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.24102867394685745, "epoch": 0.05102040816326531, "frac_reward_zero_std": 0.0, "grad_norm": 0.0077178627252578735, "learning_rate": 1.1333333333333334e-05, "loss": -0.007520838174968958, "num_tokens": 1903950.0, "reward": 0.0916748121380806, "reward_std": 0.6451828479766846, "rewards/correctness/mean": 0.5333333611488342, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.44165852665901184, "rewards/length_penalty/std": 0.277593731880188, "sampling/importance_sampling_ratio/max": 1.4403471946716309, "sampling/importance_sampling_ratio/mean": 0.9910374283790588, "sampling/importance_sampling_ratio/min": 0.6097314953804016, "sampling/sampling_logp_difference/max": 0.49473655223846436, "sampling/sampling_logp_difference/mean": 0.015411383472383022, "step": 35, "step_time": 26.316329368855804 }, { "clip_ratio/high_max": 0.0007903170286832998, "clip_ratio/high_mean": 0.0007903170286832998, "clip_ratio/low_mean": 2.127206971636042e-05, "clip_ratio/low_min": 2.127206971636042e-05, "clip_ratio/region_mean": 0.000811589095974341, "completions/clipped_ratio": 0.0, "completions/max_length": 1110.0, "completions/max_terminated_length": 1110.0, "completions/mean_length": 726.1500244140625, "completions/mean_terminated_length": 726.1500244140625, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.15926587333281836, "epoch": 0.052478134110787174, "frac_reward_zero_std": 0.0, "grad_norm": 0.007803198881447315, "learning_rate": 1.1666666666666668e-05, "loss": -0.009670584462583065, "num_tokens": 1952089.0, "reward": 0.4287679195404053, "reward_std": 0.4443022906780243, "rewards/correctness/mean": 0.7833333611488342, "rewards/correctness/std": 0.41545024514198303, "rewards/length_penalty/mean": -0.35456544160842896, "rewards/length_penalty/std": 0.11235044151544571, "sampling/importance_sampling_ratio/max": 1.724177360534668, "sampling/importance_sampling_ratio/mean": 0.9943934082984924, "sampling/importance_sampling_ratio/min": 0.6505312323570251, "sampling/sampling_logp_difference/max": 0.5447499752044678, "sampling/sampling_logp_difference/mean": 0.010538090020418167, "step": 36, "step_time": 15.591551100835204 }, { "clip_ratio/high_max": 0.0010752740296690415, "clip_ratio/high_mean": 0.0010752740296690415, "clip_ratio/low_mean": 2.604980788116033e-05, "clip_ratio/low_min": 2.604980788116033e-05, "clip_ratio/region_mean": 0.001101323839975521, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1881.0, "completions/mean_length": 903.1666870117188, "completions/mean_terminated_length": 842.9122924804688, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.27130760500828427, "epoch": 0.05393586005830904, "frac_reward_zero_std": 0.0, "grad_norm": 0.014049148187041283, "learning_rate": 1.2e-05, "loss": 0.058295562863349915, "num_tokens": 2010369.0, "reward": 0.25900065898895264, "reward_std": 0.4945225119590759, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.462124764919281, "rewards/length_penalty/mean": -0.4409993588924408, "rewards/length_penalty/std": 0.25441300868988037, "sampling/importance_sampling_ratio/max": 1.4194098711013794, "sampling/importance_sampling_ratio/mean": 0.9906554222106934, "sampling/importance_sampling_ratio/min": 0.5438868403434753, "sampling/sampling_logp_difference/max": 0.6090140342712402, "sampling/sampling_logp_difference/mean": 0.016203705221414566, "step": 37, "step_time": 25.97705299197696 }, { "clip_ratio/high_max": 0.0008416666726892194, "clip_ratio/high_mean": 0.0008416666726892194, "clip_ratio/low_mean": 0.00015480869236246994, "clip_ratio/low_min": 0.00015480869236246994, "clip_ratio/region_mean": 0.0009964753990061581, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1893.0, "completions/mean_length": 1027.61669921875, "completions/mean_terminated_length": 954.732177734375, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.2630870093901952, "epoch": 0.05539358600583091, "frac_reward_zero_std": 0.0, "grad_norm": 0.013990028761327267, "learning_rate": 1.2333333333333334e-05, "loss": 0.048009540885686874, "num_tokens": 2077336.0, "reward": -0.01843261905014515, "reward_std": 0.6465713381767273, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.5017659664154053, "rewards/length_penalty/std": 0.23699496686458588, "sampling/importance_sampling_ratio/max": 1.5592713356018066, "sampling/importance_sampling_ratio/mean": 0.9906838536262512, "sampling/importance_sampling_ratio/min": 0.4875663220882416, "sampling/sampling_logp_difference/max": 0.7183289527893066, "sampling/sampling_logp_difference/mean": 0.016107842326164246, "step": 38, "step_time": 27.35881438339129 }, { "clip_ratio/high_max": 0.000449634826509282, "clip_ratio/high_mean": 0.000449634826509282, "clip_ratio/low_mean": 8.750285148077334e-05, "clip_ratio/low_min": 8.750285148077334e-05, "clip_ratio/region_mean": 0.0005371376804153746, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1747.0, "completions/mean_length": 681.7166748046875, "completions/mean_terminated_length": 658.559326171875, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.1788865476846695, "epoch": 0.056851311953352766, "frac_reward_zero_std": 0.0, "grad_norm": 0.011713888496160507, "learning_rate": 1.2666666666666667e-05, "loss": -0.010897926986217499, "num_tokens": 2122019.0, "reward": 0.40046387910842896, "reward_std": 0.5082594156265259, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459485113620758, "rewards/length_penalty/mean": -0.3328694701194763, "rewards/length_penalty/std": 0.19904670119285583, "sampling/importance_sampling_ratio/max": 1.467110276222229, "sampling/importance_sampling_ratio/mean": 0.9937765598297119, "sampling/importance_sampling_ratio/min": 0.6686986684799194, "sampling/sampling_logp_difference/max": 0.4024217128753662, "sampling/sampling_logp_difference/mean": 0.011745237745344639, "step": 39, "step_time": 25.736718475585803 }, { "clip_ratio/high_max": 0.0010374786409859855, "clip_ratio/high_mean": 0.0010374786409859855, "clip_ratio/low_mean": 9.631955617805943e-05, "clip_ratio/low_min": 9.631955617805943e-05, "clip_ratio/region_mean": 0.0011337981753361721, "completions/clipped_ratio": 0.0, "completions/max_length": 1461.0, "completions/max_terminated_length": 1461.0, "completions/mean_length": 674.300048828125, "completions/mean_terminated_length": 674.300048828125, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.18291178345680237, "epoch": 0.05830903790087463, "frac_reward_zero_std": 0.0, "grad_norm": 0.00884868111461401, "learning_rate": 1.3000000000000001e-05, "loss": 0.001374863088130951, "num_tokens": 2167937.0, "reward": 0.520751953125, "reward_std": 0.39113694429397583, "rewards/correctness/mean": 0.8500000238418579, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.3292480409145355, "rewards/length_penalty/std": 0.1301271766424179, "sampling/importance_sampling_ratio/max": 1.3339730501174927, "sampling/importance_sampling_ratio/mean": 0.9936630725860596, "sampling/importance_sampling_ratio/min": 0.6585886478424072, "sampling/sampling_logp_difference/max": 0.41765618324279785, "sampling/sampling_logp_difference/mean": 0.011697357520461082, "step": 40, "step_time": 18.73584145028144 }, { "clip_ratio/high_max": 0.000907802406193999, "clip_ratio/high_mean": 0.000907802406193999, "clip_ratio/low_mean": 0.00012981869319143394, "clip_ratio/low_min": 0.00012981869319143394, "clip_ratio/region_mean": 0.0010376211042360712, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1965.0, "completions/mean_length": 874.4166870117188, "completions/mean_terminated_length": 812.6491088867188, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.1831328049302101, "epoch": 0.0597667638483965, "frac_reward_zero_std": 0.0, "grad_norm": 0.011611186899244785, "learning_rate": 1.3333333333333333e-05, "loss": 0.0037883776240050793, "num_tokens": 2225772.0, "reward": 0.3063720762729645, "reward_std": 0.6729326844215393, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.4269612729549408, "rewards/length_penalty/std": 0.2836941182613373, "sampling/importance_sampling_ratio/max": 1.8383829593658447, "sampling/importance_sampling_ratio/mean": 0.9938834309577942, "sampling/importance_sampling_ratio/min": 0.6318473219871521, "sampling/sampling_logp_difference/max": 0.6088863611221313, "sampling/sampling_logp_difference/mean": 0.01193917915225029, "step": 41, "step_time": 26.612672707997262 }, { "clip_ratio/high_max": 0.0006895982175289342, "clip_ratio/high_mean": 0.0006895982175289342, "clip_ratio/low_mean": 0.00018824153206272362, "clip_ratio/low_min": 0.00018824153206272362, "clip_ratio/region_mean": 0.0008778397411030406, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 2027.0, "completions/mean_length": 1014.8500366210938, "completions/mean_terminated_length": 808.219970703125, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.3642863258719444, "epoch": 0.061224489795918366, "frac_reward_zero_std": 0.0, "grad_norm": 0.016840942203998566, "learning_rate": 1.3666666666666667e-05, "loss": 0.03785286471247673, "num_tokens": 2292483.0, "reward": -0.02886556088924408, "reward_std": 0.7496117949485779, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.49553221464157104, "rewards/length_penalty/std": 0.32649514079093933, "sampling/importance_sampling_ratio/max": 1.4769392013549805, "sampling/importance_sampling_ratio/mean": 0.9869816303253174, "sampling/importance_sampling_ratio/min": 0.07880815118551254, "sampling/sampling_logp_difference/max": 2.540738821029663, "sampling/sampling_logp_difference/mean": 0.021654432639479637, "step": 42, "step_time": 27.032590586226434 }, { "clip_ratio/high_max": 0.0006048179056961089, "clip_ratio/high_mean": 0.0006048179056961089, "clip_ratio/low_mean": 0.00022036849622963928, "clip_ratio/low_min": 0.00022036849622963928, "clip_ratio/region_mean": 0.0008251863958624502, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1368.0, "completions/mean_length": 881.0667114257812, "completions/mean_terminated_length": 647.6799926757812, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.3163422793149948, "epoch": 0.06268221574344024, "frac_reward_zero_std": 0.0, "grad_norm": 0.0103360740467906, "learning_rate": 1.4e-05, "loss": 0.0021490827202796936, "num_tokens": 2350457.0, "reward": 0.3031250238418579, "reward_std": 0.6805951595306396, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.43020832538604736, "rewards/length_penalty/std": 0.27624472975730896, "sampling/importance_sampling_ratio/max": 1.4635086059570312, "sampling/importance_sampling_ratio/mean": 0.988986074924469, "sampling/importance_sampling_ratio/min": 0.60587477684021, "sampling/sampling_logp_difference/max": 0.5010819435119629, "sampling/sampling_logp_difference/mean": 0.019041329622268677, "step": 43, "step_time": 26.36771490587853 }, { "clip_ratio/high_max": 0.000841226331734409, "clip_ratio/high_mean": 0.000841226331734409, "clip_ratio/low_mean": 2.286864279691751e-05, "clip_ratio/low_min": 2.286864279691751e-05, "clip_ratio/region_mean": 0.0008640949769566456, "completions/clipped_ratio": 0.0, "completions/max_length": 1618.0, "completions/max_terminated_length": 1618.0, "completions/mean_length": 703.7166748046875, "completions/mean_terminated_length": 703.7166748046875, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.179917444785436, "epoch": 0.0641399416909621, "frac_reward_zero_std": 0.0, "grad_norm": 0.008084164001047611, "learning_rate": 1.4333333333333334e-05, "loss": 0.00908623356372118, "num_tokens": 2397870.0, "reward": 0.23972168564796448, "reward_std": 0.46271032094955444, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.3436116576194763, "rewards/length_penalty/std": 0.16231344640254974, "sampling/importance_sampling_ratio/max": 1.4407228231430054, "sampling/importance_sampling_ratio/mean": 0.9936022758483887, "sampling/importance_sampling_ratio/min": 0.6691447496414185, "sampling/sampling_logp_difference/max": 0.40175485610961914, "sampling/sampling_logp_difference/mean": 0.011499114334583282, "step": 44, "step_time": 20.7415090627037 }, { "clip_ratio/high_max": 0.00041327955599020544, "clip_ratio/high_mean": 0.00041327955599020544, "clip_ratio/low_mean": 0.00023016609460076629, "clip_ratio/low_min": 0.00023016609460076629, "clip_ratio/region_mean": 0.0006434456445276737, "completions/clipped_ratio": 0.3333333432674408, "completions/max_length": 2048.0, "completions/max_terminated_length": 1950.0, "completions/mean_length": 1233.0667724609375, "completions/mean_terminated_length": 825.6000366210938, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.2690398320555687, "epoch": 0.06559766763848396, "frac_reward_zero_std": 0.0, "grad_norm": 0.014921208843588829, "learning_rate": 1.4666666666666666e-05, "loss": 0.05788666009902954, "num_tokens": 2477384.0, "reward": -0.15208333730697632, "reward_std": 0.7774261832237244, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.6020833253860474, "rewards/length_penalty/std": 0.33607375621795654, "sampling/importance_sampling_ratio/max": 1.631103754043579, "sampling/importance_sampling_ratio/mean": 0.9906483292579651, "sampling/importance_sampling_ratio/min": 0.6383159160614014, "sampling/sampling_logp_difference/max": 0.48925697803497314, "sampling/sampling_logp_difference/mean": 0.01613265834748745, "step": 45, "step_time": 28.280902277911082 }, { "clip_ratio/high_max": 0.0004557327993097715, "clip_ratio/high_mean": 0.0004557327993097715, "clip_ratio/low_mean": 8.34783665292586e-05, "clip_ratio/low_min": 8.34783665292586e-05, "clip_ratio/region_mean": 0.0005392111658390301, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1689.0, "completions/mean_length": 601.0667114257812, "completions/mean_terminated_length": 576.5423583984375, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.2017199049393336, "epoch": 0.06705539358600583, "frac_reward_zero_std": 0.0, "grad_norm": 0.008424344472587109, "learning_rate": 1.5000000000000002e-05, "loss": 0.04910995066165924, "num_tokens": 2516748.0, "reward": 0.37317711114883423, "reward_std": 0.6525160074234009, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.29348957538604736, "rewards/length_penalty/std": 0.19386924803256989, "sampling/importance_sampling_ratio/max": 1.410577416419983, "sampling/importance_sampling_ratio/mean": 0.9931090474128723, "sampling/importance_sampling_ratio/min": 0.6503152251243591, "sampling/sampling_logp_difference/max": 0.4302980899810791, "sampling/sampling_logp_difference/mean": 0.012660418637096882, "step": 46, "step_time": 24.2268779524602 }, { "clip_ratio/high_max": 0.000743589373693491, "clip_ratio/high_mean": 0.000743589373693491, "clip_ratio/low_mean": 9.949510179770489e-05, "clip_ratio/low_min": 9.949510179770489e-05, "clip_ratio/region_mean": 0.0008430844754911959, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1772.0, "completions/mean_length": 649.2333374023438, "completions/mean_terminated_length": 575.614013671875, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.2584753731886546, "epoch": 0.06851311953352769, "frac_reward_zero_std": 0.0, "grad_norm": 0.017546163871884346, "learning_rate": 1.5333333333333334e-05, "loss": 0.017078937962651253, "num_tokens": 2560162.0, "reward": 0.43299156427383423, "reward_std": 0.5498223900794983, "rewards/correctness/mean": 0.75, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.31700846552848816, "rewards/length_penalty/std": 0.2566172182559967, "sampling/importance_sampling_ratio/max": 1.413201093673706, "sampling/importance_sampling_ratio/mean": 0.9907872080802917, "sampling/importance_sampling_ratio/min": 0.5558434724807739, "sampling/sampling_logp_difference/max": 0.587268590927124, "sampling/sampling_logp_difference/mean": 0.016103271394968033, "step": 47, "step_time": 25.70594487292692 }, { "clip_ratio/high_max": 0.0008645804336993024, "clip_ratio/high_mean": 0.0008645804336993024, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008645804336993024, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1680.0, "completions/mean_length": 594.2333374023438, "completions/mean_terminated_length": 569.5932006835938, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.24059434731801352, "epoch": 0.06997084548104957, "frac_reward_zero_std": 0.0, "grad_norm": 0.015697062015533447, "learning_rate": 1.5666666666666667e-05, "loss": 0.027538156136870384, "num_tokens": 2600006.0, "reward": 0.20984701812267303, "reward_std": 0.5217742323875427, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.29015299677848816, "rewards/length_penalty/std": 0.18181516230106354, "sampling/importance_sampling_ratio/max": 1.3966842889785767, "sampling/importance_sampling_ratio/mean": 0.9921780824661255, "sampling/importance_sampling_ratio/min": 0.6649665236473083, "sampling/sampling_logp_difference/max": 0.4080185890197754, "sampling/sampling_logp_difference/mean": 0.01451186928898096, "step": 48, "step_time": 24.447830377845094 }, { "clip_ratio/high_max": 0.00046326468388239544, "clip_ratio/high_mean": 0.00046326468388239544, "clip_ratio/low_mean": 0.00016435115200389797, "clip_ratio/low_min": 0.00016435115200389797, "clip_ratio/region_mean": 0.0006276158189090589, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1133.0, "completions/mean_length": 634.2166748046875, "completions/mean_terminated_length": 585.4655151367188, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.22726024935642877, "epoch": 0.07142857142857142, "frac_reward_zero_std": 0.0, "grad_norm": 0.010333630256354809, "learning_rate": 1.6000000000000003e-05, "loss": 0.07413791865110397, "num_tokens": 2641989.0, "reward": 0.19032390415668488, "reward_std": 0.5556463599205017, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.3096761107444763, "rewards/length_penalty/std": 0.17995549738407135, "sampling/importance_sampling_ratio/max": 1.4583498239517212, "sampling/importance_sampling_ratio/mean": 0.9922860264778137, "sampling/importance_sampling_ratio/min": 0.645832359790802, "sampling/sampling_logp_difference/max": 0.43721532821655273, "sampling/sampling_logp_difference/mean": 0.01424582302570343, "step": 49, "step_time": 25.561362505191937 }, { "clip_ratio/high_max": 0.0008010070159798488, "clip_ratio/high_mean": 0.0008010070159798488, "clip_ratio/low_mean": 5.733677365545494e-05, "clip_ratio/low_min": 5.733677365545494e-05, "clip_ratio/region_mean": 0.0008583437884226441, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1806.0, "completions/mean_length": 769.5333862304688, "completions/mean_terminated_length": 702.24560546875, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.22106604278087616, "epoch": 0.0728862973760933, "frac_reward_zero_std": 0.0, "grad_norm": 0.011128261685371399, "learning_rate": 1.6333333333333335e-05, "loss": 0.04742317646741867, "num_tokens": 2692871.0, "reward": 0.20758464932441711, "reward_std": 0.6197415590286255, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.3757486939430237, "rewards/length_penalty/std": 0.23703652620315552, "sampling/importance_sampling_ratio/max": 1.5202347040176392, "sampling/importance_sampling_ratio/mean": 0.9924477934837341, "sampling/importance_sampling_ratio/min": 0.6783851981163025, "sampling/sampling_logp_difference/max": 0.41886472702026367, "sampling/sampling_logp_difference/mean": 0.013558726757764816, "step": 50, "step_time": 25.64299608883448 }, { "clip_ratio/high_max": 0.000875128636835143, "clip_ratio/high_mean": 0.000875128636835143, "clip_ratio/low_mean": 0.0001807594186781595, "clip_ratio/low_min": 0.0001807594186781595, "clip_ratio/region_mean": 0.0010558880652145792, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1684.0, "completions/mean_length": 744.5000610351562, "completions/mean_terminated_length": 699.5516967773438, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.28909573952356976, "epoch": 0.07434402332361516, "frac_reward_zero_std": 0.0, "grad_norm": 0.021083198487758636, "learning_rate": 1.6666666666666667e-05, "loss": 0.06212671101093292, "num_tokens": 2743481.0, "reward": 0.11980795115232468, "reward_std": 0.61191326379776, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.363525390625, "rewards/length_penalty/std": 0.22039349377155304, "sampling/importance_sampling_ratio/max": 1.4389187097549438, "sampling/importance_sampling_ratio/mean": 0.9900707006454468, "sampling/importance_sampling_ratio/min": 0.5461902618408203, "sampling/sampling_logp_difference/max": 0.6047878265380859, "sampling/sampling_logp_difference/mean": 0.017100242897868156, "step": 51, "step_time": 26.10056390124373 }, { "clip_ratio/high_max": 0.0007763957012988006, "clip_ratio/high_mean": 0.0007763957012988006, "clip_ratio/low_mean": 0.00010222260607406497, "clip_ratio/low_min": 0.00010222260607406497, "clip_ratio/region_mean": 0.000878618312223504, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1858.0, "completions/mean_length": 867.300048828125, "completions/mean_terminated_length": 711.3585205078125, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.2560170814394951, "epoch": 0.07580174927113703, "frac_reward_zero_std": 0.0, "grad_norm": 0.013316703028976917, "learning_rate": 1.7e-05, "loss": 0.01976807788014412, "num_tokens": 2800329.0, "reward": 0.4098470211029053, "reward_std": 0.5818691253662109, "rewards/correctness/mean": 0.8333333134651184, "rewards/correctness/std": 0.3758230209350586, "rewards/length_penalty/mean": -0.4234863221645355, "rewards/length_penalty/std": 0.2742421329021454, "sampling/importance_sampling_ratio/max": 1.9864535331726074, "sampling/importance_sampling_ratio/mean": 0.990938663482666, "sampling/importance_sampling_ratio/min": 0.6354402899742126, "sampling/sampling_logp_difference/max": 0.68635094165802, "sampling/sampling_logp_difference/mean": 0.01604917272925377, "step": 52, "step_time": 26.209189630812034 }, { "clip_ratio/high_max": 0.0006766375542307893, "clip_ratio/high_mean": 0.0006766375542307893, "clip_ratio/low_mean": 0.00015319734544997723, "clip_ratio/low_min": 0.00015319734544997723, "clip_ratio/region_mean": 0.000829834898468107, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 1797.0, "completions/mean_length": 1292.0001220703125, "completions/mean_terminated_length": 1017.0909423828125, "completions/min_length": 489.0, "completions/min_terminated_length": 489.0, "entropy": 0.30388736476500827, "epoch": 0.07725947521865889, "frac_reward_zero_std": 0.0, "grad_norm": 0.011573869735002518, "learning_rate": 1.7333333333333336e-05, "loss": -0.0076588112860918045, "num_tokens": 2882579.0, "reward": -0.16419272124767303, "reward_std": 0.6786981821060181, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.630859375, "rewards/length_penalty/std": 0.26540708541870117, "sampling/importance_sampling_ratio/max": 1.695695400238037, "sampling/importance_sampling_ratio/mean": 0.989521861076355, "sampling/importance_sampling_ratio/min": 0.6508986353874207, "sampling/sampling_logp_difference/max": 0.5280928611755371, "sampling/sampling_logp_difference/mean": 0.017895212396979332, "step": 53, "step_time": 28.47213218314573 }, { "clip_ratio/high_max": 0.000526597427475887, "clip_ratio/high_mean": 0.000526597427475887, "clip_ratio/low_mean": 5.23866571408386e-05, "clip_ratio/low_min": 5.23866571408386e-05, "clip_ratio/region_mean": 0.0005789840773407681, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 2012.0, "completions/mean_length": 897.1000366210938, "completions/mean_terminated_length": 857.413818359375, "completions/min_length": 416.0, "completions/min_terminated_length": 416.0, "entropy": 0.2398771271109581, "epoch": 0.07871720116618076, "frac_reward_zero_std": 0.0, "grad_norm": 0.012547567486763, "learning_rate": 1.7666666666666668e-05, "loss": 0.000821845605969429, "num_tokens": 2940715.0, "reward": -0.0713704451918602, "reward_std": 0.5608754754066467, "rewards/correctness/mean": 0.36666667461395264, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.43803709745407104, "rewards/length_penalty/std": 0.21056923270225525, "sampling/importance_sampling_ratio/max": 1.5159095525741577, "sampling/importance_sampling_ratio/mean": 0.991970419883728, "sampling/importance_sampling_ratio/min": 0.6778066754341125, "sampling/sampling_logp_difference/max": 0.416015625, "sampling/sampling_logp_difference/mean": 0.014678750187158585, "step": 54, "step_time": 25.951356765115634 }, { "clip_ratio/high_max": 0.000771099810663145, "clip_ratio/high_mean": 0.000771099810663145, "clip_ratio/low_mean": 0.0002113857032478942, "clip_ratio/low_min": 0.0002113857032478942, "clip_ratio/region_mean": 0.0009824855321009334, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1933.0, "completions/mean_length": 938.9833984375, "completions/mean_terminated_length": 717.1799926757812, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.32161477704842883, "epoch": 0.08017492711370262, "frac_reward_zero_std": 0.0, "grad_norm": 0.011260569095611572, "learning_rate": 1.8e-05, "loss": 0.01119990274310112, "num_tokens": 3000564.0, "reward": 0.04151204600930214, "reward_std": 0.7691377997398376, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.45848795771598816, "rewards/length_penalty/std": 0.3181942105293274, "sampling/importance_sampling_ratio/max": 1.4323796033859253, "sampling/importance_sampling_ratio/mean": 0.9886839985847473, "sampling/importance_sampling_ratio/min": 0.6530927419662476, "sampling/sampling_logp_difference/max": 0.42603611946105957, "sampling/sampling_logp_difference/mean": 0.019134491682052612, "step": 55, "step_time": 26.1115862026345 }, { "clip_ratio/high_max": 0.0004554060918356602, "clip_ratio/high_mean": 0.0004554060918356602, "clip_ratio/low_mean": 0.00012906974249441797, "clip_ratio/low_min": 0.00012906974249441797, "clip_ratio/region_mean": 0.0005844758367553974, "completions/clipped_ratio": 0.25, "completions/max_length": 2048.0, "completions/max_terminated_length": 1948.0, "completions/mean_length": 1060.9833984375, "completions/mean_terminated_length": 731.977783203125, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "entropy": 0.22106081744035086, "epoch": 0.08163265306122448, "frac_reward_zero_std": 0.0, "grad_norm": 0.010457084514200687, "learning_rate": 1.8333333333333333e-05, "loss": -0.008733715862035751, "num_tokens": 3070233.0, "reward": -0.25139161944389343, "reward_std": 0.6637234091758728, "rewards/correctness/mean": 0.2666666805744171, "rewards/correctness/std": 0.4459485113620758, "rewards/length_penalty/mean": -0.5180582404136658, "rewards/length_penalty/std": 0.32603931427001953, "sampling/importance_sampling_ratio/max": 1.4403032064437866, "sampling/importance_sampling_ratio/mean": 0.9919553995132446, "sampling/importance_sampling_ratio/min": 0.5502684712409973, "sampling/sampling_logp_difference/max": 0.5973489284515381, "sampling/sampling_logp_difference/mean": 0.014011329971253872, "step": 56, "step_time": 27.72089922009036 }, { "clip_ratio/high_max": 0.00037454012514596496, "clip_ratio/high_mean": 0.00037454012514596496, "clip_ratio/low_mean": 0.00010635672030427183, "clip_ratio/low_min": 0.00010635672030427183, "clip_ratio/region_mean": 0.00048089684059959836, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 726.2667236328125, "completions/mean_terminated_length": 493.0196228027344, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "entropy": 0.24900235484043756, "epoch": 0.08309037900874636, "frac_reward_zero_std": 0.0, "grad_norm": 0.01727903261780739, "learning_rate": 1.866666666666667e-05, "loss": 0.00045283790677785873, "num_tokens": 3118369.0, "reward": 0.49537762999534607, "reward_std": 0.649752676486969, "rewards/correctness/mean": 0.8500000238418579, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.35462239384651184, "rewards/length_penalty/std": 0.30922913551330566, "sampling/importance_sampling_ratio/max": 1.467110276222229, "sampling/importance_sampling_ratio/mean": 0.9905803799629211, "sampling/importance_sampling_ratio/min": 0.3542850613594055, "sampling/sampling_logp_difference/max": 1.0376534461975098, "sampling/sampling_logp_difference/mean": 0.015638122335076332, "step": 57, "step_time": 25.36844819993712 }, { "clip_ratio/high_max": 0.0007073824381222948, "clip_ratio/high_mean": 0.0007073824381222948, "clip_ratio/low_mean": 2.5406504088702302e-05, "clip_ratio/low_min": 2.5406504088702302e-05, "clip_ratio/region_mean": 0.0007327889470616356, "completions/clipped_ratio": 0.0, "completions/max_length": 1676.0, "completions/max_terminated_length": 1676.0, "completions/mean_length": 591.0667114257812, "completions/mean_terminated_length": 591.0667114257812, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "entropy": 0.2171641836563746, "epoch": 0.08454810495626822, "frac_reward_zero_std": 0.0, "grad_norm": 0.010724866762757301, "learning_rate": 1.9e-05, "loss": 0.006157484836876392, "num_tokens": 3158633.0, "reward": 0.3447265923023224, "reward_std": 0.614934504032135, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.28860676288604736, "rewards/length_penalty/std": 0.18544556200504303, "sampling/importance_sampling_ratio/max": 1.6070520877838135, "sampling/importance_sampling_ratio/mean": 0.992554247379303, "sampling/importance_sampling_ratio/min": 0.561149001121521, "sampling/sampling_logp_difference/max": 0.5777688026428223, "sampling/sampling_logp_difference/mean": 0.013930361717939377, "step": 58, "step_time": 20.642161039169878 }, { "clip_ratio/high_max": 0.0011212155416918297, "clip_ratio/high_mean": 0.0011212155416918297, "clip_ratio/low_mean": 0.0001151398682850413, "clip_ratio/low_min": 0.0001151398682850413, "clip_ratio/region_mean": 0.0012363554366553824, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1823.0, "completions/mean_length": 890.3833618164062, "completions/mean_terminated_length": 785.1454467773438, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.2747861370444298, "epoch": 0.08600583090379009, "frac_reward_zero_std": 0.0, "grad_norm": 0.018310362473130226, "learning_rate": 1.9333333333333333e-05, "loss": 0.03689084202051163, "num_tokens": 3217516.0, "reward": -0.1680908352136612, "reward_std": 0.5773100852966309, "rewards/correctness/mean": 0.2666666805744171, "rewards/correctness/std": 0.4459485113620758, "rewards/length_penalty/mean": -0.4347575008869171, "rewards/length_penalty/std": 0.2805073857307434, "sampling/importance_sampling_ratio/max": 1.4319807291030884, "sampling/importance_sampling_ratio/mean": 0.9904412031173706, "sampling/importance_sampling_ratio/min": 0.5849228501319885, "sampling/sampling_logp_difference/max": 0.5362752676010132, "sampling/sampling_logp_difference/mean": 0.0173331331461668, "step": 59, "step_time": 27.54057458974421 }, { "clip_ratio/high_max": 0.001425526337698102, "clip_ratio/high_mean": 0.001425526337698102, "clip_ratio/low_mean": 0.00010533920916107793, "clip_ratio/low_min": 0.00010533920916107793, "clip_ratio/region_mean": 0.0015308655371579032, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 2044.0, "completions/mean_length": 776.11669921875, "completions/mean_terminated_length": 754.559326171875, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.2769348993897438, "epoch": 0.08746355685131195, "frac_reward_zero_std": 0.0, "grad_norm": 0.014079735614359379, "learning_rate": 1.9666666666666666e-05, "loss": 0.012713806703686714, "num_tokens": 3267503.0, "reward": 0.30437013506889343, "reward_std": 0.501975417137146, "rewards/correctness/mean": 0.6833333373069763, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.3789632022380829, "rewards/length_penalty/std": 0.2151976078748703, "sampling/importance_sampling_ratio/max": 1.6477974653244019, "sampling/importance_sampling_ratio/mean": 0.9902998805046082, "sampling/importance_sampling_ratio/min": 0.5580717325210571, "sampling/sampling_logp_difference/max": 0.5832676887512207, "sampling/sampling_logp_difference/mean": 0.0172236617654562, "step": 60, "step_time": 25.26443952252157 }, { "clip_ratio/high_max": 0.0006640923747909255, "clip_ratio/high_mean": 0.0006640923747909255, "clip_ratio/low_mean": 6.594129323881741e-05, "clip_ratio/low_min": 6.594129323881741e-05, "clip_ratio/region_mean": 0.0007300336728803813, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 2021.0, "completions/mean_length": 984.86669921875, "completions/mean_terminated_length": 888.2181396484375, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.15707450111707053, "epoch": 0.08892128279883382, "frac_reward_zero_std": 0.0, "grad_norm": 0.011188686825335026, "learning_rate": 2e-05, "loss": 0.03420940414071083, "num_tokens": 3332205.0, "reward": 0.0857747420668602, "reward_std": 0.5992581844329834, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971747398376465, "rewards/length_penalty/mean": -0.4808919131755829, "rewards/length_penalty/std": 0.2573603391647339, "sampling/importance_sampling_ratio/max": 1.447928786277771, "sampling/importance_sampling_ratio/mean": 0.9946839809417725, "sampling/importance_sampling_ratio/min": 0.5939050912857056, "sampling/sampling_logp_difference/max": 0.5210357904434204, "sampling/sampling_logp_difference/mean": 0.010414060205221176, "step": 61, "step_time": 26.998896970879287 }, { "clip_ratio/high_max": 0.0005939534506372487, "clip_ratio/high_mean": 0.0005939534506372487, "clip_ratio/low_mean": 0.00015263191501920423, "clip_ratio/low_min": 0.00015263191501920423, "clip_ratio/region_mean": 0.0007465853608058145, "completions/clipped_ratio": 0.25, "completions/max_length": 2048.0, "completions/max_terminated_length": 2044.0, "completions/mean_length": 1097.7667236328125, "completions/mean_terminated_length": 781.022216796875, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.333564189573129, "epoch": 0.09037900874635568, "frac_reward_zero_std": 0.0, "grad_norm": 0.015973957255482674, "learning_rate": 2e-05, "loss": 0.10216701030731201, "num_tokens": 3402511.0, "reward": -0.30268555879592896, "reward_std": 0.5816874504089355, "rewards/correctness/mean": 0.23333333432674408, "rewards/correctness/std": 0.42652183771133423, "rewards/length_penalty/mean": -0.5360189080238342, "rewards/length_penalty/std": 0.36388248205184937, "sampling/importance_sampling_ratio/max": 1.866272211074829, "sampling/importance_sampling_ratio/mean": 0.988468587398529, "sampling/importance_sampling_ratio/min": 0.521647572517395, "sampling/sampling_logp_difference/max": 0.6507630348205566, "sampling/sampling_logp_difference/mean": 0.019739948213100433, "step": 62, "step_time": 27.611096700653434 }, { "clip_ratio/high_max": 0.001043901705997996, "clip_ratio/high_mean": 0.001043901705997996, "clip_ratio/low_mean": 0.00015817767052794807, "clip_ratio/low_min": 0.00015817767052794807, "clip_ratio/region_mean": 0.001202079370462646, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1840.0, "completions/mean_length": 949.9500732421875, "completions/mean_terminated_length": 756.176513671875, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.3158053010702133, "epoch": 0.09183673469387756, "frac_reward_zero_std": 0.0, "grad_norm": 0.010108426213264465, "learning_rate": 2e-05, "loss": 0.035676177591085434, "num_tokens": 3464468.0, "reward": 0.16949057579040527, "reward_std": 0.6702336072921753, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.4638427793979645, "rewards/length_penalty/std": 0.27556726336479187, "sampling/importance_sampling_ratio/max": 1.4403526782989502, "sampling/importance_sampling_ratio/mean": 0.9890905022621155, "sampling/importance_sampling_ratio/min": 0.5818617343902588, "sampling/sampling_logp_difference/max": 0.5415223836898804, "sampling/sampling_logp_difference/mean": 0.018299730494618416, "step": 63, "step_time": 26.40689355507493 }, { "clip_ratio/high_max": 0.0007529740202395866, "clip_ratio/high_mean": 0.0007529740202395866, "clip_ratio/low_mean": 0.00015265539210910598, "clip_ratio/low_min": 0.00015265539210910598, "clip_ratio/region_mean": 0.0009056293977967774, "completions/clipped_ratio": 0.18333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 1879.0, "completions/mean_length": 1017.9500732421875, "completions/mean_terminated_length": 786.7142944335938, "completions/min_length": 159.0, "completions/min_terminated_length": 159.0, "entropy": 0.30281171202659607, "epoch": 0.09329446064139942, "frac_reward_zero_std": 0.0, "grad_norm": 0.012335984967648983, "learning_rate": 2e-05, "loss": 0.046371761709451675, "num_tokens": 3530505.0, "reward": -0.04704590141773224, "reward_std": 0.7366029620170593, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.4970459043979645, "rewards/length_penalty/std": 0.3209838271141052, "sampling/importance_sampling_ratio/max": 1.4508121013641357, "sampling/importance_sampling_ratio/mean": 0.9895980954170227, "sampling/importance_sampling_ratio/min": 0.1913512945175171, "sampling/sampling_logp_difference/max": 1.653644323348999, "sampling/sampling_logp_difference/mean": 0.017647288739681244, "step": 64, "step_time": 27.50379885500297 }, { "clip_ratio/high_max": 0.0010249692228777956, "clip_ratio/high_mean": 0.0010249692228777956, "clip_ratio/low_mean": 6.801817289669998e-05, "clip_ratio/low_min": 6.801817289669998e-05, "clip_ratio/region_mean": 0.0010929873872858782, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1941.0, "completions/mean_length": 745.1333618164062, "completions/mean_terminated_length": 700.2069091796875, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.23717784136533737, "epoch": 0.09475218658892129, "frac_reward_zero_std": 0.0, "grad_norm": 0.013865753076970577, "learning_rate": 2e-05, "loss": 0.03723486512899399, "num_tokens": 3579353.0, "reward": 0.43616539239883423, "reward_std": 0.44210493564605713, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.3638346493244171, "rewards/length_penalty/std": 0.2601844072341919, "sampling/importance_sampling_ratio/max": 1.4597342014312744, "sampling/importance_sampling_ratio/mean": 0.9913047552108765, "sampling/importance_sampling_ratio/min": 0.18348518013954163, "sampling/sampling_logp_difference/max": 1.695621371269226, "sampling/sampling_logp_difference/mean": 0.015431130304932594, "step": 65, "step_time": 25.39121359703131 }, { "clip_ratio/high_max": 0.0008574471285101026, "clip_ratio/high_mean": 0.0008574471285101026, "clip_ratio/low_mean": 1.902587488681699e-05, "clip_ratio/low_min": 1.902587488681699e-05, "clip_ratio/region_mean": 0.0008764730009716004, "completions/clipped_ratio": 0.0, "completions/max_length": 1282.0, "completions/max_terminated_length": 1282.0, "completions/mean_length": 842.800048828125, "completions/mean_terminated_length": 842.800048828125, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.16640034566322962, "epoch": 0.09620991253644315, "frac_reward_zero_std": 0.0, "grad_norm": 0.01003518607467413, "learning_rate": 2e-05, "loss": -0.012614110484719276, "num_tokens": 3636131.0, "reward": 0.30514323711395264, "reward_std": 0.4085974097251892, "rewards/correctness/mean": 0.7166666388511658, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.4115234315395355, "rewards/length_penalty/std": 0.11401911079883575, "sampling/importance_sampling_ratio/max": 1.459780216217041, "sampling/importance_sampling_ratio/mean": 0.9943774938583374, "sampling/importance_sampling_ratio/min": 0.6570032238960266, "sampling/sampling_logp_difference/max": 0.42006635665893555, "sampling/sampling_logp_difference/mean": 0.01115433406084776, "step": 66, "step_time": 17.655639443080872 }, { "clip_ratio/high_max": 0.0009674198275509601, "clip_ratio/high_mean": 0.0009674198275509601, "clip_ratio/low_mean": 0.0002249472114878396, "clip_ratio/low_min": 0.0002249472114878396, "clip_ratio/region_mean": 0.001192367043889438, "completions/clipped_ratio": 0.30000001192092896, "completions/max_length": 2048.0, "completions/max_terminated_length": 2024.0, "completions/mean_length": 1367.0667724609375, "completions/mean_terminated_length": 1075.2381591796875, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "entropy": 0.3390492995580037, "epoch": 0.09766763848396501, "frac_reward_zero_std": 0.0, "grad_norm": 0.015688007697463036, "learning_rate": 2e-05, "loss": 0.12193520367145538, "num_tokens": 3723895.0, "reward": -0.23417970538139343, "reward_std": 0.7415551543235779, "rewards/correctness/mean": 0.4333333373069763, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.6675130128860474, "rewards/length_penalty/std": 0.2935273051261902, "sampling/importance_sampling_ratio/max": 1.76029372215271, "sampling/importance_sampling_ratio/mean": 0.9881556034088135, "sampling/importance_sampling_ratio/min": 0.553299069404602, "sampling/sampling_logp_difference/max": 0.5918565988540649, "sampling/sampling_logp_difference/mean": 0.02006438374519348, "step": 67, "step_time": 29.594953797059134 }, { "clip_ratio/high_max": 0.0010029586652914684, "clip_ratio/high_mean": 0.0010029586652914684, "clip_ratio/low_mean": 6.584194488823414e-05, "clip_ratio/low_min": 6.584194488823414e-05, "clip_ratio/region_mean": 0.0010688006101797025, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 2028.0, "completions/mean_length": 731.1333618164062, "completions/mean_terminated_length": 685.72412109375, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.234903650979201, "epoch": 0.09912536443148688, "frac_reward_zero_std": 0.0, "grad_norm": 0.016613559797406197, "learning_rate": 2e-05, "loss": -0.02735317125916481, "num_tokens": 3772593.0, "reward": 0.37633466720581055, "reward_std": 0.5642049908638, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.3569987118244171, "rewards/length_penalty/std": 0.23382732272148132, "sampling/importance_sampling_ratio/max": 1.4509068727493286, "sampling/importance_sampling_ratio/mean": 0.991604745388031, "sampling/importance_sampling_ratio/min": 0.6131345629692078, "sampling/sampling_logp_difference/max": 0.4891709089279175, "sampling/sampling_logp_difference/mean": 0.014993232674896717, "step": 68, "step_time": 25.669897325802594 }, { "clip_ratio/high_max": 0.0006246660535301393, "clip_ratio/high_mean": 0.0006246660535301393, "clip_ratio/low_mean": 4.8398287617601454e-05, "clip_ratio/low_min": 4.8398287617601454e-05, "clip_ratio/region_mean": 0.0006730643459983791, "completions/clipped_ratio": 0.0, "completions/max_length": 1366.0, "completions/max_terminated_length": 1366.0, "completions/mean_length": 694.7667236328125, "completions/mean_terminated_length": 694.7667236328125, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.13991509626309076, "epoch": 0.10058309037900874, "frac_reward_zero_std": 0.0, "grad_norm": 0.009719469584524632, "learning_rate": 2e-05, "loss": -0.004255794454365969, "num_tokens": 3817919.0, "reward": 0.2774251401424408, "reward_std": 0.5339546203613281, "rewards/correctness/mean": 0.6166666746139526, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.33924153447151184, "rewards/length_penalty/std": 0.12143746763467789, "sampling/importance_sampling_ratio/max": 1.378197193145752, "sampling/importance_sampling_ratio/mean": 0.9951502680778503, "sampling/importance_sampling_ratio/min": 0.6503708958625793, "sampling/sampling_logp_difference/max": 0.43021249771118164, "sampling/sampling_logp_difference/mean": 0.009477663785219193, "step": 69, "step_time": 17.807427857769653 }, { "clip_ratio/high_max": 0.0006121192127466202, "clip_ratio/high_mean": 0.0006121192127466202, "clip_ratio/low_mean": 0.00014732059329010858, "clip_ratio/low_min": 0.00014732059329010858, "clip_ratio/region_mean": 0.0007594397854215155, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1808.0, "completions/mean_length": 885.0167236328125, "completions/mean_terminated_length": 844.913818359375, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.23627175887425741, "epoch": 0.10204081632653061, "frac_reward_zero_std": 0.0, "grad_norm": 0.012149685062468052, "learning_rate": 2e-05, "loss": 0.02499573305249214, "num_tokens": 3875290.0, "reward": 0.15119630098342896, "reward_std": 0.5673785209655762, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.43213704228401184, "rewards/length_penalty/std": 0.19533012807369232, "sampling/importance_sampling_ratio/max": 1.4544672966003418, "sampling/importance_sampling_ratio/mean": 0.9921490550041199, "sampling/importance_sampling_ratio/min": 0.6486742496490479, "sampling/sampling_logp_difference/max": 0.43282461166381836, "sampling/sampling_logp_difference/mean": 0.014074113219976425, "step": 70, "step_time": 27.213721254840493 }, { "clip_ratio/high_max": 0.0005193926529803624, "clip_ratio/high_mean": 0.0005193926529803624, "clip_ratio/low_mean": 7.590001526599129e-05, "clip_ratio/low_min": 7.590001526599129e-05, "clip_ratio/region_mean": 0.0005952926730969921, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1921.0, "completions/mean_length": 849.6333618164062, "completions/mean_terminated_length": 808.3103637695312, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.17722671975692114, "epoch": 0.10349854227405247, "frac_reward_zero_std": 0.0, "grad_norm": 0.009361296892166138, "learning_rate": 2e-05, "loss": 0.00986897200345993, "num_tokens": 3930718.0, "reward": 0.15180665254592896, "reward_std": 0.6519619822502136, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.4148600399494171, "rewards/length_penalty/std": 0.2246708869934082, "sampling/importance_sampling_ratio/max": 1.3928110599517822, "sampling/importance_sampling_ratio/mean": 0.9938569068908691, "sampling/importance_sampling_ratio/min": 0.6491281390190125, "sampling/sampling_logp_difference/max": 0.4321250915527344, "sampling/sampling_logp_difference/mean": 0.01106953714042902, "step": 71, "step_time": 25.902137389639392 }, { "clip_ratio/high_max": 0.0010704514182483156, "clip_ratio/high_mean": 0.0010704514182483156, "clip_ratio/low_mean": 8.437880266380186e-05, "clip_ratio/low_min": 8.437880266380186e-05, "clip_ratio/region_mean": 0.001154830203934883, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 2033.0, "completions/mean_length": 880.050048828125, "completions/mean_terminated_length": 725.79248046875, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.27359068890412647, "epoch": 0.10495626822157435, "frac_reward_zero_std": 0.0, "grad_norm": 0.016086820513010025, "learning_rate": 2e-05, "loss": 0.003041524440050125, "num_tokens": 3988201.0, "reward": 0.2702881097793579, "reward_std": 0.606598973274231, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.4297119081020355, "rewards/length_penalty/std": 0.27651119232177734, "sampling/importance_sampling_ratio/max": 1.7209787368774414, "sampling/importance_sampling_ratio/mean": 0.9903214573860168, "sampling/importance_sampling_ratio/min": 0.5492734909057617, "sampling/sampling_logp_difference/max": 0.599158763885498, "sampling/sampling_logp_difference/mean": 0.017103716731071472, "step": 72, "step_time": 26.191329712746665 }, { "clip_ratio/high_max": 0.0009247674946285164, "clip_ratio/high_mean": 0.0009247674946285164, "clip_ratio/low_mean": 7.573514206645389e-05, "clip_ratio/low_min": 7.573514206645389e-05, "clip_ratio/region_mean": 0.0010005026415456086, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1745.0, "completions/mean_length": 751.433349609375, "completions/mean_terminated_length": 729.4576416015625, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.30501004060109455, "epoch": 0.10641399416909621, "frac_reward_zero_std": 0.0, "grad_norm": 0.010819511488080025, "learning_rate": 2e-05, "loss": -0.01672491244971752, "num_tokens": 4037637.0, "reward": 0.36642253398895264, "reward_std": 0.5407053828239441, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.36691081523895264, "rewards/length_penalty/std": 0.2128191888332367, "sampling/importance_sampling_ratio/max": 1.4509228467941284, "sampling/importance_sampling_ratio/mean": 0.989236056804657, "sampling/importance_sampling_ratio/min": 0.6481517553329468, "sampling/sampling_logp_difference/max": 0.43363046646118164, "sampling/sampling_logp_difference/mean": 0.018513785675168037, "step": 73, "step_time": 25.738604004960507 }, { "clip_ratio/high_max": 0.000775772636795106, "clip_ratio/high_mean": 0.000775772636795106, "clip_ratio/low_mean": 0.0001139961532317102, "clip_ratio/low_min": 0.0001139961532317102, "clip_ratio/region_mean": 0.0008897687827508586, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1936.0, "completions/mean_length": 1033.61669921875, "completions/mean_terminated_length": 961.1607666015625, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.2437108432253202, "epoch": 0.10787172011661808, "frac_reward_zero_std": 0.0, "grad_norm": 0.012200290337204933, "learning_rate": 2e-05, "loss": -0.007034962996840477, "num_tokens": 4104524.0, "reward": 0.07863769680261612, "reward_std": 0.6565274596214294, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.5046956539154053, "rewards/length_penalty/std": 0.2536833882331848, "sampling/importance_sampling_ratio/max": 2.0870227813720703, "sampling/importance_sampling_ratio/mean": 0.9919592142105103, "sampling/importance_sampling_ratio/min": 0.5414741039276123, "sampling/sampling_logp_difference/max": 0.7357385158538818, "sampling/sampling_logp_difference/mean": 0.014488711953163147, "step": 74, "step_time": 28.6800376502797 }, { "clip_ratio/high_max": 0.0008456006908090785, "clip_ratio/high_mean": 0.0008456006908090785, "clip_ratio/low_mean": 0.0001688671254669316, "clip_ratio/low_min": 0.0001688671254669316, "clip_ratio/region_mean": 0.0010144678429545213, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1743.0, "completions/mean_length": 875.3167114257812, "completions/mean_terminated_length": 668.37255859375, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.29018878440062207, "epoch": 0.10932944606413994, "frac_reward_zero_std": 0.0, "grad_norm": 0.010037309490144253, "learning_rate": 2e-05, "loss": 0.08295391499996185, "num_tokens": 4162243.0, "reward": 0.23926596343517303, "reward_std": 0.6652383804321289, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.42740070819854736, "rewards/length_penalty/std": 0.29186713695526123, "sampling/importance_sampling_ratio/max": 1.4001084566116333, "sampling/importance_sampling_ratio/mean": 0.9903859496116638, "sampling/importance_sampling_ratio/min": 0.6442928910255432, "sampling/sampling_logp_difference/max": 0.4396018981933594, "sampling/sampling_logp_difference/mean": 0.017068399116396904, "step": 75, "step_time": 26.439316188218072 }, { "clip_ratio/high_max": 0.0007683581789024174, "clip_ratio/high_mean": 0.0007683581789024174, "clip_ratio/low_mean": 8.965843395950894e-05, "clip_ratio/low_min": 8.965843395950894e-05, "clip_ratio/region_mean": 0.0008580166225632032, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1825.0, "completions/mean_length": 862.7667236328125, "completions/mean_terminated_length": 706.2264404296875, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.32068585356076557, "epoch": 0.11078717201166181, "frac_reward_zero_std": 0.0, "grad_norm": 0.013561427593231201, "learning_rate": 2e-05, "loss": 0.03460405021905899, "num_tokens": 4218769.0, "reward": 0.09539388865232468, "reward_std": 0.6922449469566345, "rewards/correctness/mean": 0.5166666507720947, "rewards/correctness/std": 0.5039393305778503, "rewards/length_penalty/mean": -0.42127278447151184, "rewards/length_penalty/std": 0.2805083990097046, "sampling/importance_sampling_ratio/max": 1.427881121635437, "sampling/importance_sampling_ratio/mean": 0.9886918067932129, "sampling/importance_sampling_ratio/min": 0.6494015455245972, "sampling/sampling_logp_difference/max": 0.431704044342041, "sampling/sampling_logp_difference/mean": 0.018998652696609497, "step": 76, "step_time": 25.95738762873225 }, { "clip_ratio/high_max": 0.000573172643877721, "clip_ratio/high_mean": 0.000573172643877721, "clip_ratio/low_mean": 0.0001271530636586249, "clip_ratio/low_min": 0.0001271530636586249, "clip_ratio/region_mean": 0.0007003257002603883, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1871.0, "completions/mean_length": 827.6666870117188, "completions/mean_terminated_length": 612.313720703125, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.32657332470019657, "epoch": 0.11224489795918367, "frac_reward_zero_std": 0.0, "grad_norm": 0.012244322337210178, "learning_rate": 2e-05, "loss": 0.042617566883563995, "num_tokens": 4274079.0, "reward": -0.02080078236758709, "reward_std": 0.6631997227668762, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014302253723, "rewards/length_penalty/mean": -0.4041341245174408, "rewards/length_penalty/std": 0.2911378741264343, "sampling/importance_sampling_ratio/max": 1.7601795196533203, "sampling/importance_sampling_ratio/mean": 0.988547682762146, "sampling/importance_sampling_ratio/min": 0.6220017671585083, "sampling/sampling_logp_difference/max": 0.5654158592224121, "sampling/sampling_logp_difference/mean": 0.019659103825688362, "step": 77, "step_time": 26.511172500206158 }, { "clip_ratio/high_max": 0.0006040186854079366, "clip_ratio/high_mean": 0.0006040186854079366, "clip_ratio/low_mean": 0.00011882732602922867, "clip_ratio/low_min": 0.00011882732602922867, "clip_ratio/region_mean": 0.0007228460065865269, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 2011.0, "completions/mean_length": 1037.86669921875, "completions/mean_terminated_length": 904.4528198242188, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "entropy": 0.22266371299823126, "epoch": 0.11370262390670553, "frac_reward_zero_std": 0.0, "grad_norm": 0.0142426248639822, "learning_rate": 2e-05, "loss": 0.01979541964828968, "num_tokens": 4342531.0, "reward": -0.15677084028720856, "reward_std": 0.6428810358047485, "rewards/correctness/mean": 0.3499999940395355, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.5067708492279053, "rewards/length_penalty/std": 0.26104095578193665, "sampling/importance_sampling_ratio/max": 1.7160686254501343, "sampling/importance_sampling_ratio/mean": 0.9918795228004456, "sampling/importance_sampling_ratio/min": 0.6340979933738708, "sampling/sampling_logp_difference/max": 0.5400359630584717, "sampling/sampling_logp_difference/mean": 0.01398763433098793, "step": 78, "step_time": 27.06211461359635 }, { "clip_ratio/high_max": 0.0004470327755067653, "clip_ratio/high_mean": 0.0004470327755067653, "clip_ratio/low_mean": 0.00013647254066502987, "clip_ratio/low_min": 0.00013647254066502987, "clip_ratio/region_mean": 0.0005835053161717951, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 1006.3333740234375, "completions/mean_terminated_length": 822.5098266601562, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.20668349415063858, "epoch": 0.1151603498542274, "frac_reward_zero_std": 0.0, "grad_norm": 0.009284527972340584, "learning_rate": 2e-05, "loss": 0.06438212096691132, "num_tokens": 4409821.0, "reward": 0.1752929836511612, "reward_std": 0.7006873488426208, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.4913736879825592, "rewards/length_penalty/std": 0.26470717787742615, "sampling/importance_sampling_ratio/max": 1.4403252601623535, "sampling/importance_sampling_ratio/mean": 0.9931297898292542, "sampling/importance_sampling_ratio/min": 0.6608903408050537, "sampling/sampling_logp_difference/max": 0.4141674041748047, "sampling/sampling_logp_difference/mean": 0.012824874371290207, "step": 79, "step_time": 27.364660371094942 }, { "clip_ratio/high_max": 0.000855635866173543, "clip_ratio/high_mean": 0.000855635866173543, "clip_ratio/low_mean": 0.00013942622414712483, "clip_ratio/low_min": 0.00013942622414712483, "clip_ratio/region_mean": 0.0009950621169991791, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 777.9000244140625, "completions/mean_terminated_length": 687.1785888671875, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.2723602006832759, "epoch": 0.11661807580174927, "frac_reward_zero_std": 0.0, "grad_norm": 0.016341570764780045, "learning_rate": 2e-05, "loss": 0.07856422662734985, "num_tokens": 4461305.0, "reward": 0.47016602754592896, "reward_std": 0.568079948425293, "rewards/correctness/mean": 0.8500000238418579, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.37983399629592896, "rewards/length_penalty/std": 0.270346462726593, "sampling/importance_sampling_ratio/max": 1.515631914138794, "sampling/importance_sampling_ratio/mean": 0.990748405456543, "sampling/importance_sampling_ratio/min": 0.6570536494255066, "sampling/sampling_logp_difference/max": 0.41998958587646484, "sampling/sampling_logp_difference/mean": 0.016672134399414062, "step": 80, "step_time": 26.63881313498132 }, { "clip_ratio/high_max": 0.0008328702727643152, "clip_ratio/high_mean": 0.0008328702727643152, "clip_ratio/low_mean": 0.0001242530840196802, "clip_ratio/low_min": 0.0001242530840196802, "clip_ratio/region_mean": 0.0009571233725485703, "completions/clipped_ratio": 0.31666669249534607, "completions/max_length": 2048.0, "completions/max_terminated_length": 2000.0, "completions/mean_length": 959.8167114257812, "completions/mean_terminated_length": 455.53656005859375, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.23871558656295142, "epoch": 0.11807580174927114, "frac_reward_zero_std": 0.0, "grad_norm": 0.013187786564230919, "learning_rate": 2e-05, "loss": 0.06366457045078278, "num_tokens": 4523414.0, "reward": -0.0019938151817768812, "reward_std": 0.8292409181594849, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.46866047382354736, "rewards/length_penalty/std": 0.396953821182251, "sampling/importance_sampling_ratio/max": 1.4598079919815063, "sampling/importance_sampling_ratio/mean": 0.9917196035385132, "sampling/importance_sampling_ratio/min": 0.5851331353187561, "sampling/sampling_logp_difference/max": 0.5359158515930176, "sampling/sampling_logp_difference/mean": 0.014803586527705193, "step": 81, "step_time": 26.944744786946103 }, { "clip_ratio/high_max": 0.0011079669323711034, "clip_ratio/high_mean": 0.0011079669323711034, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011079669323711034, "completions/clipped_ratio": 0.0, "completions/max_length": 922.0, "completions/max_terminated_length": 922.0, "completions/mean_length": 443.0500183105469, "completions/mean_terminated_length": 443.0500183105469, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.19146856168905893, "epoch": 0.119533527696793, "frac_reward_zero_std": 0.0, "grad_norm": 0.0076869698241353035, "learning_rate": 2e-05, "loss": -0.0073247989639639854, "num_tokens": 4553937.0, "reward": 0.43366700410842896, "reward_std": 0.4700399339199066, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.21633300185203552, "rewards/length_penalty/std": 0.08693579584360123, "sampling/importance_sampling_ratio/max": 1.4495785236358643, "sampling/importance_sampling_ratio/mean": 0.9935380816459656, "sampling/importance_sampling_ratio/min": 0.6671428680419922, "sampling/sampling_logp_difference/max": 0.4047510623931885, "sampling/sampling_logp_difference/mean": 0.012338114902377129, "step": 82, "step_time": 11.948872851207852 }, { "clip_ratio/high_max": 0.0008603990378711993, "clip_ratio/high_mean": 0.0008603990378711993, "clip_ratio/low_mean": 0.00012837099590493986, "clip_ratio/low_min": 0.00012837099590493986, "clip_ratio/region_mean": 0.0009887700386267777, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1752.0, "completions/mean_length": 942.3167114257812, "completions/mean_terminated_length": 665.8958740234375, "completions/min_length": 433.0, "completions/min_terminated_length": 433.0, "entropy": 0.3316035866737366, "epoch": 0.12099125364431487, "frac_reward_zero_std": 0.0, "grad_norm": 0.012608137913048267, "learning_rate": 2e-05, "loss": 0.054517023265361786, "num_tokens": 4614906.0, "reward": 0.13988444209098816, "reward_std": 0.6998077630996704, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.46011555194854736, "rewards/length_penalty/std": 0.2946597635746002, "sampling/importance_sampling_ratio/max": 1.5668257474899292, "sampling/importance_sampling_ratio/mean": 0.9886726140975952, "sampling/importance_sampling_ratio/min": 0.6418333649635315, "sampling/sampling_logp_difference/max": 0.4490518569946289, "sampling/sampling_logp_difference/mean": 0.019341254606842995, "step": 83, "step_time": 26.819837241899222 }, { "clip_ratio/high_max": 0.0005293477273274524, "clip_ratio/high_mean": 0.0005293477273274524, "clip_ratio/low_mean": 9.641703218221664e-05, "clip_ratio/low_min": 9.641703218221664e-05, "clip_ratio/region_mean": 0.000625764759509669, "completions/clipped_ratio": 0.0, "completions/max_length": 1947.0, "completions/max_terminated_length": 1947.0, "completions/mean_length": 796.6000366210938, "completions/mean_terminated_length": 796.6000366210938, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.20828798661629358, "epoch": 0.12244897959183673, "frac_reward_zero_std": 0.0, "grad_norm": 0.013983390294015408, "learning_rate": 2e-05, "loss": -0.004879172891378403, "num_tokens": 4667622.0, "reward": 0.19436849653720856, "reward_std": 0.5332536697387695, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.38896483182907104, "rewards/length_penalty/std": 0.25310996174812317, "sampling/importance_sampling_ratio/max": 1.6624352931976318, "sampling/importance_sampling_ratio/mean": 0.9927626252174377, "sampling/importance_sampling_ratio/min": 0.6399871706962585, "sampling/sampling_logp_difference/max": 0.5082836151123047, "sampling/sampling_logp_difference/mean": 0.012836500070989132, "step": 84, "step_time": 24.730451047653332 }, { "clip_ratio/high_max": 0.00036194647691445425, "clip_ratio/high_mean": 0.00036194647691445425, "clip_ratio/low_mean": 7.499000639654696e-05, "clip_ratio/low_min": 7.499000639654696e-05, "clip_ratio/region_mean": 0.0004369364833110012, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1934.0, "completions/mean_length": 654.2000122070312, "completions/mean_terminated_length": 470.11322021484375, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.2506712128718694, "epoch": 0.1239067055393586, "frac_reward_zero_std": 0.0, "grad_norm": 0.01700645312666893, "learning_rate": 2e-05, "loss": 0.0787142887711525, "num_tokens": 4710704.0, "reward": 0.4972330927848816, "reward_std": 0.6686694025993347, "rewards/correctness/mean": 0.8166666626930237, "rewards/correctness/std": 0.39020493626594543, "rewards/length_penalty/mean": -0.3194335997104645, "rewards/length_penalty/std": 0.293500691652298, "sampling/importance_sampling_ratio/max": 1.4508010149002075, "sampling/importance_sampling_ratio/mean": 0.991134524345398, "sampling/importance_sampling_ratio/min": 0.5592718720436096, "sampling/sampling_logp_difference/max": 0.5811195373535156, "sampling/sampling_logp_difference/mean": 0.015687232837080956, "step": 85, "step_time": 24.8261538639199 }, { "clip_ratio/high_max": 0.0007935117804057276, "clip_ratio/high_mean": 0.0007935117804057276, "clip_ratio/low_mean": 0.0002459599963913206, "clip_ratio/low_min": 0.0002459599963913206, "clip_ratio/region_mean": 0.0010394717889236442, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1464.0, "completions/mean_length": 867.0333862304688, "completions/mean_terminated_length": 630.8399658203125, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.2866263687610626, "epoch": 0.12536443148688048, "frac_reward_zero_std": 0.0, "grad_norm": 0.01707608811557293, "learning_rate": 2e-05, "loss": 0.08202922344207764, "num_tokens": 4767436.0, "reward": 0.20997722446918488, "reward_std": 0.6878211498260498, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.4233561158180237, "rewards/length_penalty/std": 0.2928031086921692, "sampling/importance_sampling_ratio/max": 1.4199771881103516, "sampling/importance_sampling_ratio/mean": 0.9901440143585205, "sampling/importance_sampling_ratio/min": 0.2758060097694397, "sampling/sampling_logp_difference/max": 1.288057565689087, "sampling/sampling_logp_difference/mean": 0.01677057519555092, "step": 86, "step_time": 26.44699452095665 }, { "clip_ratio/high_max": 0.000700932631540733, "clip_ratio/high_mean": 0.000700932631540733, "clip_ratio/low_mean": 0.0001029938454545724, "clip_ratio/low_min": 0.0001029938454545724, "clip_ratio/region_mean": 0.0008039264551674327, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 2039.0, "completions/mean_length": 953.550048828125, "completions/mean_terminated_length": 854.0545043945312, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.21370024979114532, "epoch": 0.12682215743440234, "frac_reward_zero_std": 0.0, "grad_norm": 0.008855806663632393, "learning_rate": 2e-05, "loss": -0.005549025721848011, "num_tokens": 4829639.0, "reward": 0.26773276925086975, "reward_std": 0.6256986260414124, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.4656005799770355, "rewards/length_penalty/std": 0.24547874927520752, "sampling/importance_sampling_ratio/max": 1.4311411380767822, "sampling/importance_sampling_ratio/mean": 0.992712140083313, "sampling/importance_sampling_ratio/min": 0.6615297198295593, "sampling/sampling_logp_difference/max": 0.41320037841796875, "sampling/sampling_logp_difference/mean": 0.013411559164524078, "step": 87, "step_time": 26.37262093811296 }, { "clip_ratio/high_max": 0.0004775548926166569, "clip_ratio/high_mean": 0.0004775548926166569, "clip_ratio/low_mean": 9.304770234545383e-05, "clip_ratio/low_min": 9.304770234545383e-05, "clip_ratio/region_mean": 0.0005706025925367916, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1775.0, "completions/mean_length": 971.1834106445312, "completions/mean_terminated_length": 701.9791870117188, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.3344749758640925, "epoch": 0.1282798833819242, "frac_reward_zero_std": 0.0, "grad_norm": 0.010389057919383049, "learning_rate": 2e-05, "loss": 0.000678627286106348, "num_tokens": 4895050.0, "reward": -0.05754394829273224, "reward_std": 0.6811898350715637, "rewards/correctness/mean": 0.4166666567325592, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.47421061992645264, "rewards/length_penalty/std": 0.29730790853500366, "sampling/importance_sampling_ratio/max": 2.342852830886841, "sampling/importance_sampling_ratio/mean": 0.98845374584198, "sampling/importance_sampling_ratio/min": 0.6082951426506042, "sampling/sampling_logp_difference/max": 0.8513693809509277, "sampling/sampling_logp_difference/mean": 0.019974932074546814, "step": 88, "step_time": 26.829658325761557 }, { "clip_ratio/high_max": 0.0011431698706777145, "clip_ratio/high_mean": 0.0011431698706777145, "clip_ratio/low_mean": 8.953552848349015e-05, "clip_ratio/low_min": 8.953552848349015e-05, "clip_ratio/region_mean": 0.0012327053894599278, "completions/clipped_ratio": 0.0, "completions/max_length": 1462.0, "completions/max_terminated_length": 1462.0, "completions/mean_length": 723.7833862304688, "completions/mean_terminated_length": 723.7833862304688, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.20457501461108527, "epoch": 0.12973760932944606, "frac_reward_zero_std": 0.0, "grad_norm": 0.00969801563769579, "learning_rate": 2e-05, "loss": -0.008164532482624054, "num_tokens": 4943227.0, "reward": 0.513256847858429, "reward_std": 0.3979385793209076, "rewards/correctness/mean": 0.8666666746139526, "rewards/correctness/std": 0.34280332922935486, "rewards/length_penalty/mean": -0.3534098267555237, "rewards/length_penalty/std": 0.15708020329475403, "sampling/importance_sampling_ratio/max": 1.7208794355392456, "sampling/importance_sampling_ratio/mean": 0.9932364821434021, "sampling/importance_sampling_ratio/min": 0.5768768191337585, "sampling/sampling_logp_difference/max": 0.5501265525817871, "sampling/sampling_logp_difference/mean": 0.012802796438336372, "step": 89, "step_time": 19.591808903263882 }, { "clip_ratio/high_max": 0.001249313597024108, "clip_ratio/high_mean": 0.001249313597024108, "clip_ratio/low_mean": 0.00016507423909691474, "clip_ratio/low_min": 0.00016507423909691474, "clip_ratio/region_mean": 0.0014143877973159154, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1938.0, "completions/mean_length": 758.7500610351562, "completions/mean_terminated_length": 690.8947143554688, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.3826302985350291, "epoch": 0.13119533527696792, "frac_reward_zero_std": 0.0, "grad_norm": 0.01774374209344387, "learning_rate": 2e-05, "loss": 0.0025574974715709686, "num_tokens": 4994272.0, "reward": 0.4628499448299408, "reward_std": 0.5554989576339722, "rewards/correctness/mean": 0.8333333134651184, "rewards/correctness/std": 0.3758230209350586, "rewards/length_penalty/mean": -0.3704833984375, "rewards/length_penalty/std": 0.24367263913154602, "sampling/importance_sampling_ratio/max": 1.4403685331344604, "sampling/importance_sampling_ratio/mean": 0.9864087104797363, "sampling/importance_sampling_ratio/min": 0.644669771194458, "sampling/sampling_logp_difference/max": 0.43901705741882324, "sampling/sampling_logp_difference/mean": 0.023107297718524933, "step": 90, "step_time": 26.096014055190608 }, { "clip_ratio/high_max": 0.0011761372346275796, "clip_ratio/high_mean": 0.0011761372346275796, "clip_ratio/low_mean": 0.00015822998830117285, "clip_ratio/low_min": 0.00015822998830117285, "clip_ratio/region_mean": 0.0013343672423313062, "completions/clipped_ratio": 0.0, "completions/max_length": 1663.0, "completions/max_terminated_length": 1663.0, "completions/mean_length": 617.683349609375, "completions/mean_terminated_length": 617.683349609375, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.23400658120711645, "epoch": 0.1326530612244898, "frac_reward_zero_std": 0.0, "grad_norm": 0.012364847585558891, "learning_rate": 2e-05, "loss": 0.013461099937558174, "num_tokens": 5034703.0, "reward": 0.4983968436717987, "reward_std": 0.4498032331466675, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.30160319805145264, "rewards/length_penalty/std": 0.14153972268104553, "sampling/importance_sampling_ratio/max": 1.3339767456054688, "sampling/importance_sampling_ratio/mean": 0.9915993213653564, "sampling/importance_sampling_ratio/min": 0.6506576538085938, "sampling/sampling_logp_difference/max": 0.42977166175842285, "sampling/sampling_logp_difference/mean": 0.014409150928258896, "step": 91, "step_time": 20.228758974932134 }, { "clip_ratio/high_max": 0.0011599833572593827, "clip_ratio/high_mean": 0.0011599833572593827, "clip_ratio/low_mean": 7.72899159831771e-05, "clip_ratio/low_min": 7.72899159831771e-05, "clip_ratio/region_mean": 0.001237273245351389, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1521.0, "completions/mean_length": 877.933349609375, "completions/mean_terminated_length": 643.9199829101562, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.24022843688726425, "epoch": 0.13411078717201166, "frac_reward_zero_std": 0.0, "grad_norm": 0.009864171966910362, "learning_rate": 2e-05, "loss": 0.018639594316482544, "num_tokens": 5092429.0, "reward": 0.3713216483592987, "reward_std": 0.6548762917518616, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.42867839336395264, "rewards/length_penalty/std": 0.28254130482673645, "sampling/importance_sampling_ratio/max": 1.8417235612869263, "sampling/importance_sampling_ratio/mean": 0.9913477301597595, "sampling/importance_sampling_ratio/min": 0.5614640116691589, "sampling/sampling_logp_difference/max": 0.6107019186019897, "sampling/sampling_logp_difference/mean": 0.014954106882214546, "step": 92, "step_time": 26.760359250940382 }, { "clip_ratio/high_max": 0.0007670078193768859, "clip_ratio/high_mean": 0.0007670078193768859, "clip_ratio/low_mean": 6.530691219571357e-05, "clip_ratio/low_min": 6.530691219571357e-05, "clip_ratio/region_mean": 0.0008323147339979187, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1655.0, "completions/mean_length": 649.5000610351562, "completions/mean_terminated_length": 625.796630859375, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.18720209846893945, "epoch": 0.13556851311953352, "frac_reward_zero_std": 0.0, "grad_norm": 0.012682660482823849, "learning_rate": 2e-05, "loss": 0.010452114045619965, "num_tokens": 5135879.0, "reward": 0.23286134004592896, "reward_std": 0.5095825791358948, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.317138671875, "rewards/length_penalty/std": 0.184276282787323, "sampling/importance_sampling_ratio/max": 1.467110276222229, "sampling/importance_sampling_ratio/mean": 0.9933242201805115, "sampling/importance_sampling_ratio/min": 0.6245324015617371, "sampling/sampling_logp_difference/max": 0.4707520008087158, "sampling/sampling_logp_difference/mean": 0.012335265055298805, "step": 93, "step_time": 24.67962512979284 }, { "clip_ratio/high_max": 0.0006980508672616755, "clip_ratio/high_mean": 0.0006980508672616755, "clip_ratio/low_mean": 0.00016362378179716566, "clip_ratio/low_min": 0.00016362378179716566, "clip_ratio/region_mean": 0.0008616746442082027, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1909.0, "completions/mean_length": 895.050048828125, "completions/mean_terminated_length": 717.673095703125, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.3231743698318799, "epoch": 0.13702623906705538, "frac_reward_zero_std": 0.0, "grad_norm": 0.011201799847185612, "learning_rate": 2e-05, "loss": 0.032010000199079514, "num_tokens": 5194552.0, "reward": 0.19629721343517303, "reward_std": 0.6891112327575684, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.4370361268520355, "rewards/length_penalty/std": 0.2695411741733551, "sampling/importance_sampling_ratio/max": 1.4182049036026, "sampling/importance_sampling_ratio/mean": 0.9891645312309265, "sampling/importance_sampling_ratio/min": 0.6431248188018799, "sampling/sampling_logp_difference/max": 0.44141650199890137, "sampling/sampling_logp_difference/mean": 0.018724149093031883, "step": 94, "step_time": 26.034434655681252 }, { "clip_ratio/high_max": 0.0008400467486353591, "clip_ratio/high_mean": 0.0008400467486353591, "clip_ratio/low_mean": 9.523039382960026e-05, "clip_ratio/low_min": 9.523039382960026e-05, "clip_ratio/region_mean": 0.0009352771448902786, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1336.0, "completions/mean_length": 526.933349609375, "completions/mean_terminated_length": 501.1525573730469, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.24195876717567444, "epoch": 0.13848396501457727, "frac_reward_zero_std": 0.0, "grad_norm": 0.012131198309361935, "learning_rate": 2e-05, "loss": 0.00936015136539936, "num_tokens": 5230318.0, "reward": 0.5760416984558105, "reward_std": 0.5255236625671387, "rewards/correctness/mean": 0.8333333134651184, "rewards/correctness/std": 0.3758230209350586, "rewards/length_penalty/mean": -0.25729167461395264, "rewards/length_penalty/std": 0.17985829710960388, "sampling/importance_sampling_ratio/max": 1.3941067457199097, "sampling/importance_sampling_ratio/mean": 0.9913172125816345, "sampling/importance_sampling_ratio/min": 0.6241722106933594, "sampling/sampling_logp_difference/max": 0.4713289737701416, "sampling/sampling_logp_difference/mean": 0.015595714561641216, "step": 95, "step_time": 24.485636703902856 }, { "clip_ratio/high_max": 0.0008672485249311043, "clip_ratio/high_mean": 0.0008672485249311043, "clip_ratio/low_mean": 0.00011490157824785759, "clip_ratio/low_min": 0.00011490157824785759, "clip_ratio/region_mean": 0.0009821500910523657, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1849.0, "completions/mean_length": 695.6666870117188, "completions/mean_terminated_length": 599.0714721679688, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.3335142781337102, "epoch": 0.13994169096209913, "frac_reward_zero_std": 0.0, "grad_norm": 0.010053694248199463, "learning_rate": 2e-05, "loss": 0.007662919815629721, "num_tokens": 5276378.0, "reward": 0.04365234449505806, "reward_std": 0.5776897072792053, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014004230499, "rewards/length_penalty/mean": -0.3396809995174408, "rewards/length_penalty/std": 0.2500426769256592, "sampling/importance_sampling_ratio/max": 1.3981760740280151, "sampling/importance_sampling_ratio/mean": 0.9888298511505127, "sampling/importance_sampling_ratio/min": 0.6447350978851318, "sampling/sampling_logp_difference/max": 0.4389157295227051, "sampling/sampling_logp_difference/mean": 0.019536839798092842, "step": 96, "step_time": 25.759525523055345 }, { "clip_ratio/high_max": 0.0008366378121233234, "clip_ratio/high_mean": 0.0008366378121233234, "clip_ratio/low_mean": 6.1031064736501626e-05, "clip_ratio/low_min": 6.1031064736501626e-05, "clip_ratio/region_mean": 0.0008976688744345059, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 2036.0, "completions/mean_length": 1042.300048828125, "completions/mean_terminated_length": 970.46435546875, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.2069831813375155, "epoch": 0.141399416909621, "frac_reward_zero_std": 0.0, "grad_norm": 0.013124002143740654, "learning_rate": 2e-05, "loss": 0.02402597665786743, "num_tokens": 5342656.0, "reward": 0.3410644829273224, "reward_std": 0.5162246227264404, "rewards/correctness/mean": 0.8500000238418579, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.5089355707168579, "rewards/length_penalty/std": 0.25521671772003174, "sampling/importance_sampling_ratio/max": 1.467110276222229, "sampling/importance_sampling_ratio/mean": 0.9927272796630859, "sampling/importance_sampling_ratio/min": 0.6346305012702942, "sampling/sampling_logp_difference/max": 0.4547123312950134, "sampling/sampling_logp_difference/mean": 0.012942169792950153, "step": 97, "step_time": 26.65434269607067 }, { "clip_ratio/high_max": 0.0005699570174328983, "clip_ratio/high_mean": 0.0005699570174328983, "clip_ratio/low_mean": 2.466577764910956e-05, "clip_ratio/low_min": 2.466577764910956e-05, "clip_ratio/region_mean": 0.0005946227999326462, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1804.0, "completions/mean_length": 646.1000366210938, "completions/mean_terminated_length": 597.7586059570312, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.19671294589837393, "epoch": 0.14285714285714285, "frac_reward_zero_std": 0.0, "grad_norm": 0.011593547649681568, "learning_rate": 2e-05, "loss": 0.03229053318500519, "num_tokens": 5385912.0, "reward": 0.5845215320587158, "reward_std": 0.4700944423675537, "rewards/correctness/mean": 0.8999999761581421, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.31547850370407104, "rewards/length_penalty/std": 0.2142878621816635, "sampling/importance_sampling_ratio/max": 1.5550326108932495, "sampling/importance_sampling_ratio/mean": 0.9931764006614685, "sampling/importance_sampling_ratio/min": 0.6677953004837036, "sampling/sampling_logp_difference/max": 0.44149649143218994, "sampling/sampling_logp_difference/mean": 0.012409945018589497, "step": 98, "step_time": 25.017244498012587 }, { "clip_ratio/high_max": 0.0007077873306116089, "clip_ratio/high_mean": 0.0007077873306116089, "clip_ratio/low_mean": 0.0002881991070656416, "clip_ratio/low_min": 0.0002881991070656416, "clip_ratio/region_mean": 0.0009959864595051233, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1926.0, "completions/mean_length": 1104.300048828125, "completions/mean_terminated_length": 868.375, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.34258048236370087, "epoch": 0.14431486880466474, "frac_reward_zero_std": 0.0, "grad_norm": 0.015076699666678905, "learning_rate": 2e-05, "loss": 0.022467955946922302, "num_tokens": 5456770.0, "reward": 0.02745768427848816, "reward_std": 0.7769045233726501, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.5392090082168579, "rewards/length_penalty/std": 0.3212955892086029, "sampling/importance_sampling_ratio/max": 1.7142083644866943, "sampling/importance_sampling_ratio/mean": 0.9883654117584229, "sampling/importance_sampling_ratio/min": 0.6409561634063721, "sampling/sampling_logp_difference/max": 0.5389513969421387, "sampling/sampling_logp_difference/mean": 0.02026149071753025, "step": 99, "step_time": 28.01622262224555 }, { "clip_ratio/high_max": 0.0006826348374791754, "clip_ratio/high_mean": 0.0006826348374791754, "clip_ratio/low_mean": 0.0002501061159515909, "clip_ratio/low_min": 0.0002501061159515909, "clip_ratio/region_mean": 0.0009327409352408722, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1678.0, "completions/mean_length": 847.1000366210938, "completions/mean_terminated_length": 688.4906005859375, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.3057691554228465, "epoch": 0.1457725947521866, "frac_reward_zero_std": 0.0, "grad_norm": 0.012148505076766014, "learning_rate": 2e-05, "loss": 0.044301412999629974, "num_tokens": 5513056.0, "reward": 0.00304361991584301, "reward_std": 0.6464745998382568, "rewards/correctness/mean": 0.4166666567325592, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.41362303495407104, "rewards/length_penalty/std": 0.26829221844673157, "sampling/importance_sampling_ratio/max": 1.4854363203048706, "sampling/importance_sampling_ratio/mean": 0.9892914295196533, "sampling/importance_sampling_ratio/min": 0.5826400518417358, "sampling/sampling_logp_difference/max": 0.5401856899261475, "sampling/sampling_logp_difference/mean": 0.018160300329327583, "step": 100, "step_time": 25.87883016304113 }, { "clip_ratio/high_max": 0.0008231871761381626, "clip_ratio/high_mean": 0.0008231871761381626, "clip_ratio/low_mean": 0.00010636348937017222, "clip_ratio/low_min": 0.00010636348937017222, "clip_ratio/region_mean": 0.000929550655807058, "completions/clipped_ratio": 0.0, "completions/max_length": 1616.0, "completions/max_terminated_length": 1616.0, "completions/mean_length": 684.6000366210938, "completions/mean_terminated_length": 684.6000366210938, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.21437226235866547, "epoch": 0.14723032069970846, "frac_reward_zero_std": 0.0, "grad_norm": 0.012079442851245403, "learning_rate": 2e-05, "loss": -0.00972786732017994, "num_tokens": 5558272.0, "reward": 0.39905601739883423, "reward_std": 0.47673743963241577, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.33427733182907104, "rewards/length_penalty/std": 0.17992880940437317, "sampling/importance_sampling_ratio/max": 1.4278860092163086, "sampling/importance_sampling_ratio/mean": 0.9925777912139893, "sampling/importance_sampling_ratio/min": 0.652592658996582, "sampling/sampling_logp_difference/max": 0.4268021583557129, "sampling/sampling_logp_difference/mean": 0.013579588383436203, "step": 101, "step_time": 20.694484879029915 }, { "clip_ratio/high_max": 0.000667893347175171, "clip_ratio/high_mean": 0.000667893347175171, "clip_ratio/low_mean": 0.00019019936007680371, "clip_ratio/low_min": 0.00019019936007680371, "clip_ratio/region_mean": 0.0008580927096772939, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 1866.0, "completions/mean_length": 1047.416748046875, "completions/mean_terminated_length": 847.2999877929688, "completions/min_length": 456.0, "completions/min_terminated_length": 456.0, "entropy": 0.22797627747058868, "epoch": 0.14868804664723032, "frac_reward_zero_std": 0.0, "grad_norm": 0.015009243972599506, "learning_rate": 2e-05, "loss": 0.043725211173295975, "num_tokens": 5625017.0, "reward": 0.03856608271598816, "reward_std": 0.6842026710510254, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.5114338994026184, "rewards/length_penalty/std": 0.2826176881790161, "sampling/importance_sampling_ratio/max": 1.7557878494262695, "sampling/importance_sampling_ratio/mean": 0.9918825030326843, "sampling/importance_sampling_ratio/min": 0.48450207710266113, "sampling/sampling_logp_difference/max": 0.7246335744857788, "sampling/sampling_logp_difference/mean": 0.014599970541894436, "step": 102, "step_time": 26.755916321650147 }, { "clip_ratio/high_max": 0.0009433443968494734, "clip_ratio/high_mean": 0.0009433443968494734, "clip_ratio/low_mean": 4.761904710903764e-05, "clip_ratio/low_min": 4.761904710903764e-05, "clip_ratio/region_mean": 0.0009909634439585109, "completions/clipped_ratio": 0.0, "completions/max_length": 574.0, "completions/max_terminated_length": 574.0, "completions/mean_length": 353.1000061035156, "completions/mean_terminated_length": 353.1000061035156, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 0.1891030122836431, "epoch": 0.15014577259475217, "frac_reward_zero_std": 0.0, "grad_norm": 0.008301123045384884, "learning_rate": 2e-05, "loss": 0.001749962568283081, "num_tokens": 5649073.0, "reward": 0.7442545890808105, "reward_std": 0.30027633905410767, "rewards/correctness/mean": 0.9166666865348816, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.17241211235523224, "rewards/length_penalty/std": 0.05345404893159866, "sampling/importance_sampling_ratio/max": 1.443211317062378, "sampling/importance_sampling_ratio/mean": 0.9940182566642761, "sampling/importance_sampling_ratio/min": 0.6817989945411682, "sampling/sampling_logp_difference/max": 0.38302040100097656, "sampling/sampling_logp_difference/mean": 0.012686309404671192, "step": 103, "step_time": 8.264996593119577 }, { "clip_ratio/high_max": 0.0017504769057268277, "clip_ratio/high_mean": 0.0017504769057268277, "clip_ratio/low_mean": 0.00015876545148785226, "clip_ratio/low_min": 0.00015876545148785226, "clip_ratio/region_mean": 0.0019092423899564892, "completions/clipped_ratio": 0.21666668355464935, "completions/max_length": 2048.0, "completions/max_terminated_length": 1798.0, "completions/mean_length": 1092.5667724609375, "completions/mean_terminated_length": 828.2978515625, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.2928629020849864, "epoch": 0.15160349854227406, "frac_reward_zero_std": 0.0, "grad_norm": 0.011957842856645584, "learning_rate": 2e-05, "loss": 0.022531531751155853, "num_tokens": 5721567.0, "reward": -0.0668131560087204, "reward_std": 0.6712607145309448, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.5334798097610474, "rewards/length_penalty/std": 0.31666451692581177, "sampling/importance_sampling_ratio/max": 1.4848307371139526, "sampling/importance_sampling_ratio/mean": 0.9893854260444641, "sampling/importance_sampling_ratio/min": 0.43411290645599365, "sampling/sampling_logp_difference/max": 0.8344506025314331, "sampling/sampling_logp_difference/mean": 0.018202045932412148, "step": 104, "step_time": 29.20547517691739 }, { "clip_ratio/high_max": 0.0007943510960709924, "clip_ratio/high_mean": 0.0007943510960709924, "clip_ratio/low_mean": 0.00014659620016270006, "clip_ratio/low_min": 0.00014659620016270006, "clip_ratio/region_mean": 0.0009409472986590117, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1329.0, "completions/mean_length": 641.7500610351562, "completions/mean_terminated_length": 617.915283203125, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.22921649614969888, "epoch": 0.15306122448979592, "frac_reward_zero_std": 0.0, "grad_norm": 0.00924638845026493, "learning_rate": 2e-05, "loss": 0.003329500090330839, "num_tokens": 5767162.0, "reward": 0.16997885704040527, "reward_std": 0.6293733716011047, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.3133544921875, "rewards/length_penalty/std": 0.171682208776474, "sampling/importance_sampling_ratio/max": 1.4909831285476685, "sampling/importance_sampling_ratio/mean": 0.9919578433036804, "sampling/importance_sampling_ratio/min": 0.6197322607040405, "sampling/sampling_logp_difference/max": 0.4784677028656006, "sampling/sampling_logp_difference/mean": 0.014817951247096062, "step": 105, "step_time": 25.627611643401906 }, { "clip_ratio/high_max": 0.001136383895451824, "clip_ratio/high_mean": 0.001136383895451824, "clip_ratio/low_mean": 1.6441419575130567e-05, "clip_ratio/low_min": 1.6441419575130567e-05, "clip_ratio/region_mean": 0.00115282532836621, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1839.0, "completions/mean_length": 922.0167236328125, "completions/mean_terminated_length": 841.5892944335938, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.2763413538535436, "epoch": 0.15451895043731778, "frac_reward_zero_std": 0.0, "grad_norm": 0.015668585896492004, "learning_rate": 2e-05, "loss": 0.04884755611419678, "num_tokens": 5827893.0, "reward": 0.449796587228775, "reward_std": 0.48367294669151306, "rewards/correctness/mean": 0.8999999761581421, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.45020344853401184, "rewards/length_penalty/std": 0.2587897777557373, "sampling/importance_sampling_ratio/max": 1.450933814048767, "sampling/importance_sampling_ratio/mean": 0.990755021572113, "sampling/importance_sampling_ratio/min": 0.6321578025817871, "sampling/sampling_logp_difference/max": 0.4586162567138672, "sampling/sampling_logp_difference/mean": 0.016498787328600883, "step": 106, "step_time": 27.20574454916641 }, { "clip_ratio/high_max": 0.0007279627946748709, "clip_ratio/high_mean": 0.0007279627946748709, "clip_ratio/low_mean": 6.508324440801516e-05, "clip_ratio/low_min": 6.508324440801516e-05, "clip_ratio/region_mean": 0.0007930460415082052, "completions/clipped_ratio": 0.0, "completions/max_length": 901.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 504.16668701171875, "completions/mean_terminated_length": 504.16668701171875, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.16326933602492014, "epoch": 0.15597667638483964, "frac_reward_zero_std": 0.0, "grad_norm": 0.008952063508331776, "learning_rate": 2e-05, "loss": -0.004902808461338282, "num_tokens": 5862133.0, "reward": 0.38715821504592896, "reward_std": 0.5228502154350281, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.2461751252412796, "rewards/length_penalty/std": 0.08678169548511505, "sampling/importance_sampling_ratio/max": 1.7018791437149048, "sampling/importance_sampling_ratio/mean": 0.9946082830429077, "sampling/importance_sampling_ratio/min": 0.6620524525642395, "sampling/sampling_logp_difference/max": 0.5317330360412598, "sampling/sampling_logp_difference/mean": 0.011036716401576996, "step": 107, "step_time": 11.78904084279202 }, { "clip_ratio/high_max": 0.0008622771662582333, "clip_ratio/high_mean": 0.0008622771662582333, "clip_ratio/low_mean": 0.00011014389747288078, "clip_ratio/low_min": 0.00011014389747288078, "clip_ratio/region_mean": 0.0009724210685817525, "completions/clipped_ratio": 0.0, "completions/max_length": 828.0, "completions/max_terminated_length": 828.0, "completions/mean_length": 476.13336181640625, "completions/mean_terminated_length": 476.13336181640625, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 0.24200636645158133, "epoch": 0.15743440233236153, "frac_reward_zero_std": 0.0, "grad_norm": 0.009640194475650787, "learning_rate": 2e-05, "loss": -0.00044576090294867754, "num_tokens": 5895831.0, "reward": 0.10084635764360428, "reward_std": 0.4789772629737854, "rewards/correctness/mean": 0.3333333432674408, "rewards/correctness/std": 0.4753826856613159, "rewards/length_penalty/mean": -0.23248697817325592, "rewards/length_penalty/std": 0.07821419835090637, "sampling/importance_sampling_ratio/max": 1.4278913736343384, "sampling/importance_sampling_ratio/mean": 0.9917312264442444, "sampling/importance_sampling_ratio/min": 0.6590301990509033, "sampling/sampling_logp_difference/max": 0.41698598861694336, "sampling/sampling_logp_difference/mean": 0.015516233630478382, "step": 108, "step_time": 11.562631891807541 }, { "clip_ratio/high_max": 0.0007829671449144371, "clip_ratio/high_mean": 0.0007829671449144371, "clip_ratio/low_mean": 0.0001908329916962733, "clip_ratio/low_min": 0.0001908329916962733, "clip_ratio/region_mean": 0.0009738001293347528, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1752.0, "completions/mean_length": 852.36669921875, "completions/mean_terminated_length": 789.4385986328125, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.29062982896963757, "epoch": 0.1588921282798834, "frac_reward_zero_std": 0.0, "grad_norm": 0.010587343014776707, "learning_rate": 2e-05, "loss": 0.021413404494524002, "num_tokens": 5954923.0, "reward": 0.3338053524494171, "reward_std": 0.5650517344474792, "rewards/correctness/mean": 0.75, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.4161946475505829, "rewards/length_penalty/std": 0.20770102739334106, "sampling/importance_sampling_ratio/max": 1.5148905515670776, "sampling/importance_sampling_ratio/mean": 0.9898743033409119, "sampling/importance_sampling_ratio/min": 0.4112025797367096, "sampling/sampling_logp_difference/max": 0.8886693716049194, "sampling/sampling_logp_difference/mean": 0.01754608564078808, "step": 109, "step_time": 27.17084173578769 }, { "clip_ratio/high_max": 0.0009141297972140213, "clip_ratio/high_mean": 0.0009141297972140213, "clip_ratio/low_mean": 0.00023761014502573138, "clip_ratio/low_min": 0.00023761014502573138, "clip_ratio/region_mean": 0.0011517399252625182, "completions/clipped_ratio": 0.21666668355464935, "completions/max_length": 2048.0, "completions/max_terminated_length": 2007.0, "completions/mean_length": 984.7500610351562, "completions/mean_terminated_length": 690.6595458984375, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 0.4111553480227788, "epoch": 0.16034985422740525, "frac_reward_zero_std": 0.0, "grad_norm": 0.015779634937644005, "learning_rate": 2e-05, "loss": 0.09138652682304382, "num_tokens": 6018318.0, "reward": 0.08583170920610428, "reward_std": 0.7085700631141663, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.4808349609375, "rewards/length_penalty/std": 0.30886977910995483, "sampling/importance_sampling_ratio/max": 1.432719111442566, "sampling/importance_sampling_ratio/mean": 0.9857117533683777, "sampling/importance_sampling_ratio/min": 0.606770396232605, "sampling/sampling_logp_difference/max": 0.49960482120513916, "sampling/sampling_logp_difference/mean": 0.02312241494655609, "step": 110, "step_time": 26.642415746115148 }, { "clip_ratio/high_max": 0.0006274778036943948, "clip_ratio/high_mean": 0.0006274778036943948, "clip_ratio/low_mean": 6.698924213803063e-05, "clip_ratio/low_min": 6.698924213803063e-05, "clip_ratio/region_mean": 0.0006944670434071062, "completions/clipped_ratio": 0.0, "completions/max_length": 987.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 452.41668701171875, "completions/mean_terminated_length": 452.41668701171875, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.19472946474949518, "epoch": 0.1618075801749271, "frac_reward_zero_std": 0.0, "grad_norm": 0.009772730059921741, "learning_rate": 2e-05, "loss": -0.005842759273946285, "num_tokens": 6050023.0, "reward": 0.6957601308822632, "reward_std": 0.3222009241580963, "rewards/correctness/mean": 0.9166666865348816, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.2209065705537796, "rewards/length_penalty/std": 0.09787384420633316, "sampling/importance_sampling_ratio/max": 1.421118140220642, "sampling/importance_sampling_ratio/mean": 0.9931695461273193, "sampling/importance_sampling_ratio/min": 0.6763921976089478, "sampling/sampling_logp_difference/max": 0.39098215103149414, "sampling/sampling_logp_difference/mean": 0.012520139105618, "step": 111, "step_time": 13.254243111237884 }, { "clip_ratio/high_max": 0.0005047594289256571, "clip_ratio/high_mean": 0.0005047594289256571, "clip_ratio/low_mean": 8.132254515658133e-05, "clip_ratio/low_min": 8.132254515658133e-05, "clip_ratio/region_mean": 0.0005860819825708555, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1977.0, "completions/mean_length": 1124.2667236328125, "completions/mean_terminated_length": 1040.2908935546875, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.23988608767588934, "epoch": 0.16326530612244897, "frac_reward_zero_std": 0.0, "grad_norm": 0.014338881708681583, "learning_rate": 2e-05, "loss": 0.05090166628360748, "num_tokens": 6121959.0, "reward": -0.0989583358168602, "reward_std": 0.6507073640823364, "rewards/correctness/mean": 0.44999998807907104, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.5489583611488342, "rewards/length_penalty/std": 0.23871323466300964, "sampling/importance_sampling_ratio/max": 1.4403581619262695, "sampling/importance_sampling_ratio/mean": 0.991570770740509, "sampling/importance_sampling_ratio/min": 0.48501986265182495, "sampling/sampling_logp_difference/max": 0.7235654592514038, "sampling/sampling_logp_difference/mean": 0.015152319334447384, "step": 112, "step_time": 27.166342918062583 }, { "clip_ratio/high_max": 0.0011074609986584012, "clip_ratio/high_mean": 0.0011074609986584012, "clip_ratio/low_mean": 4.443851988374566e-05, "clip_ratio/low_min": 4.443851988374566e-05, "clip_ratio/region_mean": 0.001151899530668743, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 757.2667236328125, "completions/mean_terminated_length": 712.7586059570312, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.2371274953087171, "epoch": 0.16472303206997085, "frac_reward_zero_std": 0.0, "grad_norm": 0.012095651589334011, "learning_rate": 2e-05, "loss": 0.046309418976306915, "num_tokens": 6171975.0, "reward": 0.3469075560569763, "reward_std": 0.4822749197483063, "rewards/correctness/mean": 0.7166666388511658, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.36975911259651184, "rewards/length_penalty/std": 0.18612240254878998, "sampling/importance_sampling_ratio/max": 1.420120120048523, "sampling/importance_sampling_ratio/mean": 0.9917676448822021, "sampling/importance_sampling_ratio/min": 0.5245876908302307, "sampling/sampling_logp_difference/max": 0.645142674446106, "sampling/sampling_logp_difference/mean": 0.014648997224867344, "step": 113, "step_time": 26.096502298722044 }, { "clip_ratio/high_max": 0.0010038233691981684, "clip_ratio/high_mean": 0.0010038233691981684, "clip_ratio/low_mean": 9.734569539432414e-05, "clip_ratio/low_min": 9.734569539432414e-05, "clip_ratio/region_mean": 0.0011011690367013216, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1854.0, "completions/mean_length": 923.4833984375, "completions/mean_terminated_length": 750.4807739257812, "completions/min_length": 462.0, "completions/min_terminated_length": 462.0, "entropy": 0.30406248321135837, "epoch": 0.1661807580174927, "frac_reward_zero_std": 0.0, "grad_norm": 0.010896090418100357, "learning_rate": 2e-05, "loss": -0.005914328619837761, "num_tokens": 6231604.0, "reward": 0.3490804135799408, "reward_std": 0.6274470090866089, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.45091959834098816, "rewards/length_penalty/std": 0.249985471367836, "sampling/importance_sampling_ratio/max": 1.4471254348754883, "sampling/importance_sampling_ratio/mean": 0.9893328547477722, "sampling/importance_sampling_ratio/min": 0.6689406037330627, "sampling/sampling_logp_difference/max": 0.40206003189086914, "sampling/sampling_logp_difference/mean": 0.018006250262260437, "step": 114, "step_time": 26.676251277094707 }, { "clip_ratio/high_max": 0.00043013242005448166, "clip_ratio/high_mean": 0.00043013242005448166, "clip_ratio/low_mean": 0.000206737793632783, "clip_ratio/low_min": 0.000206737793632783, "clip_ratio/region_mean": 0.0006368702112619454, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 1806.0, "completions/mean_length": 1120.283447265625, "completions/mean_terminated_length": 782.9318237304688, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.3369167347749074, "epoch": 0.16763848396501457, "frac_reward_zero_std": 0.0, "grad_norm": 0.01141158677637577, "learning_rate": 2e-05, "loss": 0.044828686863183975, "num_tokens": 6304291.0, "reward": -0.11368001997470856, "reward_std": 0.7972869873046875, "rewards/correctness/mean": 0.4333333373069763, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.5470133423805237, "rewards/length_penalty/std": 0.345553994178772, "sampling/importance_sampling_ratio/max": 1.459780216217041, "sampling/importance_sampling_ratio/mean": 0.9883002638816833, "sampling/importance_sampling_ratio/min": 0.5899421572685242, "sampling/sampling_logp_difference/max": 0.5277307033538818, "sampling/sampling_logp_difference/mean": 0.01965204067528248, "step": 115, "step_time": 27.795288962312043 }, { "clip_ratio/high_max": 0.0004979478107998148, "clip_ratio/high_mean": 0.0004979478107998148, "clip_ratio/low_mean": 0.00015590013936161995, "clip_ratio/low_min": 0.00015590013936161995, "clip_ratio/region_mean": 0.0006538479428854771, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 1970.0, "completions/mean_length": 1126.416748046875, "completions/mean_terminated_length": 791.2954711914062, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "entropy": 0.3682697092493375, "epoch": 0.16909620991253643, "frac_reward_zero_std": 0.0, "grad_norm": 0.016534850001335144, "learning_rate": 2e-05, "loss": 0.0898706391453743, "num_tokens": 6376686.0, "reward": 0.01665852963924408, "reward_std": 0.7991631627082825, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.5500081181526184, "rewards/length_penalty/std": 0.3267991542816162, "sampling/importance_sampling_ratio/max": 1.4670318365097046, "sampling/importance_sampling_ratio/mean": 0.9871171712875366, "sampling/importance_sampling_ratio/min": 0.6514776945114136, "sampling/sampling_logp_difference/max": 0.4285120964050293, "sampling/sampling_logp_difference/mean": 0.02166234888136387, "step": 116, "step_time": 27.74404573487118 }, { "clip_ratio/high_max": 0.0008037452256151786, "clip_ratio/high_mean": 0.0008037452256151786, "clip_ratio/low_mean": 2.661556451736639e-05, "clip_ratio/low_min": 2.661556451736639e-05, "clip_ratio/region_mean": 0.0008303607901325449, "completions/clipped_ratio": 0.0, "completions/max_length": 1107.0, "completions/max_terminated_length": 1107.0, "completions/mean_length": 606.6500244140625, "completions/mean_terminated_length": 606.6500244140625, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.2057554523150126, "epoch": 0.17055393586005832, "frac_reward_zero_std": 0.0, "grad_norm": 0.010343342088162899, "learning_rate": 2e-05, "loss": 0.009249605238437653, "num_tokens": 6418125.0, "reward": 0.5037841796875, "reward_std": 0.39930668473243713, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.29621583223342896, "rewards/length_penalty/std": 0.09691675752401352, "sampling/importance_sampling_ratio/max": 1.4326097965240479, "sampling/importance_sampling_ratio/mean": 0.9929855465888977, "sampling/importance_sampling_ratio/min": 0.657730758190155, "sampling/sampling_logp_difference/max": 0.4189596176147461, "sampling/sampling_logp_difference/mean": 0.013125832192599773, "step": 117, "step_time": 14.536963875871152 }, { "clip_ratio/high_max": 0.0009456396231447192, "clip_ratio/high_mean": 0.0009456396231447192, "clip_ratio/low_mean": 6.059137134191891e-05, "clip_ratio/low_min": 6.059137134191891e-05, "clip_ratio/region_mean": 0.0010062309932739784, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1401.0, "completions/mean_length": 577.9500122070312, "completions/mean_terminated_length": 500.5789489746094, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.25233660886685055, "epoch": 0.17201166180758018, "frac_reward_zero_std": 0.0, "grad_norm": 0.0110774589702487, "learning_rate": 2e-05, "loss": 0.04032396152615547, "num_tokens": 6457122.0, "reward": 0.41779786348342896, "reward_std": 0.6189420819282532, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.2822021543979645, "rewards/length_penalty/std": 0.2023358941078186, "sampling/importance_sampling_ratio/max": 1.4071640968322754, "sampling/importance_sampling_ratio/mean": 0.991232693195343, "sampling/importance_sampling_ratio/min": 0.5602762699127197, "sampling/sampling_logp_difference/max": 0.5793251991271973, "sampling/sampling_logp_difference/mean": 0.016044238582253456, "step": 118, "step_time": 24.539006631355733 }, { "clip_ratio/high_max": 0.0005012461285029227, "clip_ratio/high_mean": 0.0005012461285029227, "clip_ratio/low_mean": 0.00011166577314725146, "clip_ratio/low_min": 0.00011166577314725146, "clip_ratio/region_mean": 0.0006129119089261318, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1698.0, "completions/mean_length": 696.4000244140625, "completions/mean_terminated_length": 488.4615478515625, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 0.22835732996463776, "epoch": 0.17346938775510204, "frac_reward_zero_std": 0.0, "grad_norm": 0.01016679685562849, "learning_rate": 2e-05, "loss": 0.010078591294586658, "num_tokens": 6504116.0, "reward": 0.3099609613418579, "reward_std": 0.6975182890892029, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.34003907442092896, "rewards/length_penalty/std": 0.2913699746131897, "sampling/importance_sampling_ratio/max": 1.4403685331344604, "sampling/importance_sampling_ratio/mean": 0.991732120513916, "sampling/importance_sampling_ratio/min": 0.5591676831245422, "sampling/sampling_logp_difference/max": 0.5813058614730835, "sampling/sampling_logp_difference/mean": 0.01446867547929287, "step": 119, "step_time": 26.685395759763196 }, { "clip_ratio/high_max": 0.0009717244538478553, "clip_ratio/high_mean": 0.0009717244538478553, "clip_ratio/low_mean": 0.00020052308173035271, "clip_ratio/low_min": 0.00020052308173035271, "clip_ratio/region_mean": 0.0011722475367908676, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 2031.0, "completions/mean_length": 923.050048828125, "completions/mean_terminated_length": 798.0555419921875, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 0.23362419505914053, "epoch": 0.1749271137026239, "frac_reward_zero_std": 0.0, "grad_norm": 0.015248598530888557, "learning_rate": 2e-05, "loss": 0.05209886655211449, "num_tokens": 6563929.0, "reward": 0.03262532874941826, "reward_std": 0.6359132528305054, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.4507080018520355, "rewards/length_penalty/std": 0.2893427610397339, "sampling/importance_sampling_ratio/max": 2.494029998779297, "sampling/importance_sampling_ratio/mean": 0.9919755458831787, "sampling/importance_sampling_ratio/min": 0.6522249579429626, "sampling/sampling_logp_difference/max": 0.9138998985290527, "sampling/sampling_logp_difference/mean": 0.014302060939371586, "step": 120, "step_time": 26.535727652022615 }, { "clip_ratio/high_max": 0.0004296866354707163, "clip_ratio/high_mean": 0.0004296866354707163, "clip_ratio/low_mean": 0.00015267804944111654, "clip_ratio/low_min": 0.00015267804944111654, "clip_ratio/region_mean": 0.0005823646982510885, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1350.0, "completions/mean_length": 783.7833862304688, "completions/mean_terminated_length": 560.686279296875, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.4176936869819959, "epoch": 0.17638483965014579, "frac_reward_zero_std": 0.0, "grad_norm": 0.008688746020197868, "learning_rate": 2e-05, "loss": 0.001725086010992527, "num_tokens": 6614916.0, "reward": 0.08395996689796448, "reward_std": 0.6422682404518127, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.3827067017555237, "rewards/length_penalty/std": 0.27486416697502136, "sampling/importance_sampling_ratio/max": 1.4164466857910156, "sampling/importance_sampling_ratio/mean": 0.9856937527656555, "sampling/importance_sampling_ratio/min": 0.5565100908279419, "sampling/sampling_logp_difference/max": 0.5860700607299805, "sampling/sampling_logp_difference/mean": 0.023755677044391632, "step": 121, "step_time": 25.770300252130255 }, { "clip_ratio/high_max": 0.0004953260383141848, "clip_ratio/high_mean": 0.0004953260383141848, "clip_ratio/low_mean": 0.00020123962652481472, "clip_ratio/low_min": 0.00020123962652481472, "clip_ratio/region_mean": 0.000696565669689638, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1481.0, "completions/mean_length": 756.4667358398438, "completions/mean_terminated_length": 639.0545043945312, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.3642074267069499, "epoch": 0.17784256559766765, "frac_reward_zero_std": 0.0, "grad_norm": 0.01703622192144394, "learning_rate": 2e-05, "loss": 0.025002527981996536, "num_tokens": 6665144.0, "reward": 0.21396484971046448, "reward_std": 0.6749217510223389, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.3693684935569763, "rewards/length_penalty/std": 0.24041447043418884, "sampling/importance_sampling_ratio/max": 1.4403629302978516, "sampling/importance_sampling_ratio/mean": 0.9874864816665649, "sampling/importance_sampling_ratio/min": 0.5955517888069153, "sampling/sampling_logp_difference/max": 0.5182669162750244, "sampling/sampling_logp_difference/mean": 0.02108372375369072, "step": 122, "step_time": 26.300538864918053 }, { "clip_ratio/high_max": 0.0011148687820726384, "clip_ratio/high_mean": 0.0011148687820726384, "clip_ratio/low_mean": 9.008175159882133e-05, "clip_ratio/low_min": 9.008175159882133e-05, "clip_ratio/region_mean": 0.0012049505409474175, "completions/clipped_ratio": 0.0, "completions/max_length": 1987.0, "completions/max_terminated_length": 1987.0, "completions/mean_length": 766.4500122070312, "completions/mean_terminated_length": 766.4500122070312, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.198521355787913, "epoch": 0.1793002915451895, "frac_reward_zero_std": 0.0, "grad_norm": 0.011132443323731422, "learning_rate": 2e-05, "loss": 0.0012619979679584503, "num_tokens": 6716791.0, "reward": 0.49242353439331055, "reward_std": 0.38170745968818665, "rewards/correctness/mean": 0.8666666746139526, "rewards/correctness/std": 0.34280332922935486, "rewards/length_penalty/mean": -0.3742431700229645, "rewards/length_penalty/std": 0.16473336517810822, "sampling/importance_sampling_ratio/max": 1.439562439918518, "sampling/importance_sampling_ratio/mean": 0.992904782295227, "sampling/importance_sampling_ratio/min": 0.4723667502403259, "sampling/sampling_logp_difference/max": 0.7499996423721313, "sampling/sampling_logp_difference/mean": 0.012830211780965328, "step": 123, "step_time": 24.510049313073978 }, { "clip_ratio/high_max": 0.0006655274919467047, "clip_ratio/high_mean": 0.0006655274919467047, "clip_ratio/low_mean": 0.0003246031459032868, "clip_ratio/low_min": 0.0003246031459032868, "clip_ratio/region_mean": 0.0009901306378499914, "completions/clipped_ratio": 0.21666668355464935, "completions/max_length": 2048.0, "completions/max_terminated_length": 1711.0, "completions/mean_length": 991.7333984375, "completions/mean_terminated_length": 699.574462890625, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.3197648997108142, "epoch": 0.18075801749271136, "frac_reward_zero_std": 0.0, "grad_norm": 0.018542077392339706, "learning_rate": 2e-05, "loss": 0.07707426697015762, "num_tokens": 6780235.0, "reward": 0.01575521007180214, "reward_std": 0.7430059313774109, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.48424479365348816, "rewards/length_penalty/std": 0.32906633615493774, "sampling/importance_sampling_ratio/max": 1.648996353149414, "sampling/importance_sampling_ratio/mean": 0.9885635375976562, "sampling/importance_sampling_ratio/min": 0.6289244890213013, "sampling/sampling_logp_difference/max": 0.5001667737960815, "sampling/sampling_logp_difference/mean": 0.019025621935725212, "step": 124, "step_time": 26.444706220878288 }, { "clip_ratio/high_max": 0.0005606148139728854, "clip_ratio/high_mean": 0.0005606148139728854, "clip_ratio/low_mean": 0.0001834169537081228, "clip_ratio/low_min": 0.0001834169537081228, "clip_ratio/region_mean": 0.0007440317519164333, "completions/clipped_ratio": 0.23333334922790527, "completions/max_length": 2048.0, "completions/max_terminated_length": 1928.0, "completions/mean_length": 1131.7000732421875, "completions/mean_terminated_length": 852.8261108398438, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.2989233136177063, "epoch": 0.18221574344023322, "frac_reward_zero_std": 0.0, "grad_norm": 0.01351092103868723, "learning_rate": 2e-05, "loss": 0.08028208464384079, "num_tokens": 6852627.0, "reward": 0.0974121168255806, "reward_std": 0.7621957063674927, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.5525878667831421, "rewards/length_penalty/std": 0.3241298198699951, "sampling/importance_sampling_ratio/max": 1.4517446756362915, "sampling/importance_sampling_ratio/mean": 0.9894800186157227, "sampling/importance_sampling_ratio/min": 0.6503453254699707, "sampling/sampling_logp_difference/max": 0.4302518367767334, "sampling/sampling_logp_difference/mean": 0.017761768773198128, "step": 125, "step_time": 27.731713354354724 }, { "clip_ratio/high_max": 0.0004232441254619819, "clip_ratio/high_mean": 0.0004232441254619819, "clip_ratio/low_mean": 0.00024141892936313525, "clip_ratio/low_min": 0.00024141892936313525, "clip_ratio/region_mean": 0.0006646630693770325, "completions/clipped_ratio": 0.18333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 1277.916748046875, "completions/mean_terminated_length": 1105.040771484375, "completions/min_length": 419.0, "completions/min_terminated_length": 419.0, "entropy": 0.2783774783213933, "epoch": 0.1836734693877551, "frac_reward_zero_std": 0.0, "grad_norm": 0.012387122958898544, "learning_rate": 2e-05, "loss": 0.035095829516649246, "num_tokens": 6936512.0, "reward": -0.24064943194389343, "reward_std": 0.7253720164299011, "rewards/correctness/mean": 0.38333332538604736, "rewards/correctness/std": 0.4903014302253723, "rewards/length_penalty/mean": -0.6239827275276184, "rewards/length_penalty/std": 0.27273300290107727, "sampling/importance_sampling_ratio/max": 1.6612882614135742, "sampling/importance_sampling_ratio/mean": 0.990386426448822, "sampling/importance_sampling_ratio/min": 0.5785393118858337, "sampling/sampling_logp_difference/max": 0.5472488403320312, "sampling/sampling_logp_difference/mean": 0.016780639067292213, "step": 126, "step_time": 28.712742294883355 }, { "clip_ratio/high_max": 0.0009337971811570848, "clip_ratio/high_mean": 0.0009337971811570848, "clip_ratio/low_mean": 0.0002482736502618839, "clip_ratio/low_min": 0.0002482736502618839, "clip_ratio/region_mean": 0.0011820707974645, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1819.0, "completions/mean_length": 949.2000732421875, "completions/mean_terminated_length": 780.1538696289062, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.4010750949382782, "epoch": 0.18513119533527697, "frac_reward_zero_std": 0.0, "grad_norm": 0.013337875716388226, "learning_rate": 2e-05, "loss": 0.016871880739927292, "num_tokens": 6998284.0, "reward": 0.08652344346046448, "reward_std": 0.6886231303215027, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.4634765684604645, "rewards/length_penalty/std": 0.2748017907142639, "sampling/importance_sampling_ratio/max": 1.4107288122177124, "sampling/importance_sampling_ratio/mean": 0.9860550165176392, "sampling/importance_sampling_ratio/min": 0.6588606238365173, "sampling/sampling_logp_difference/max": 0.41724324226379395, "sampling/sampling_logp_difference/mean": 0.023269442841410637, "step": 127, "step_time": 27.239234886830673 }, { "clip_ratio/high_max": 0.0004995747658540495, "clip_ratio/high_mean": 0.0004995747658540495, "clip_ratio/low_mean": 0.00010617072014914204, "clip_ratio/low_min": 0.00010617072014914204, "clip_ratio/region_mean": 0.0006057454884285107, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1957.0, "completions/mean_length": 951.4000244140625, "completions/mean_terminated_length": 829.5555419921875, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.21109230816364288, "epoch": 0.18658892128279883, "frac_reward_zero_std": 0.0, "grad_norm": 0.012693856842815876, "learning_rate": 2e-05, "loss": -0.014684738591313362, "num_tokens": 7061468.0, "reward": -0.28121745586395264, "reward_std": 0.5295966267585754, "rewards/correctness/mean": 0.18333333730697632, "rewards/correctness/std": 0.39020493626594543, "rewards/length_penalty/mean": -0.46455079317092896, "rewards/length_penalty/std": 0.2660280466079712, "sampling/importance_sampling_ratio/max": 1.4671049118041992, "sampling/importance_sampling_ratio/mean": 0.9926087260246277, "sampling/importance_sampling_ratio/min": 0.5823831558227539, "sampling/sampling_logp_difference/max": 0.5406267642974854, "sampling/sampling_logp_difference/mean": 0.013477679342031479, "step": 128, "step_time": 28.212278296938166 }, { "clip_ratio/high_max": 0.0009754830462043174, "clip_ratio/high_mean": 0.0009754830462043174, "clip_ratio/low_mean": 8.977870796419059e-05, "clip_ratio/low_min": 8.977870796419059e-05, "clip_ratio/region_mean": 0.0010652617468925503, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1830.0, "completions/mean_length": 774.7500610351562, "completions/mean_terminated_length": 659.0, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.34729481240113574, "epoch": 0.1880466472303207, "frac_reward_zero_std": 0.0, "grad_norm": 0.009247354231774807, "learning_rate": 2e-05, "loss": -0.005573976784944534, "num_tokens": 7112833.0, "reward": -0.17829591035842896, "reward_std": 0.524168074131012, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.3782958984375, "rewards/length_penalty/std": 0.24723808467388153, "sampling/importance_sampling_ratio/max": 1.4727504253387451, "sampling/importance_sampling_ratio/mean": 0.9879551529884338, "sampling/importance_sampling_ratio/min": 0.6780223250389099, "sampling/sampling_logp_difference/max": 0.38857507705688477, "sampling/sampling_logp_difference/mean": 0.0211569145321846, "step": 129, "step_time": 25.679753988282755 }, { "clip_ratio/high_max": 0.0009580096278417235, "clip_ratio/high_mean": 0.0009580096278417235, "clip_ratio/low_mean": 6.613554554254127e-05, "clip_ratio/low_min": 6.613554554254127e-05, "clip_ratio/region_mean": 0.0010241451770222436, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 2010.0, "completions/mean_length": 875.36669921875, "completions/mean_terminated_length": 791.607177734375, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.29244259744882584, "epoch": 0.18950437317784258, "frac_reward_zero_std": 0.0, "grad_norm": 0.015463539399206638, "learning_rate": 2e-05, "loss": 0.045666784048080444, "num_tokens": 7171605.0, "reward": 0.2392415553331375, "reward_std": 0.6368379592895508, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.4274251163005829, "rewards/length_penalty/std": 0.26480501890182495, "sampling/importance_sampling_ratio/max": 1.4454598426818848, "sampling/importance_sampling_ratio/mean": 0.9897004961967468, "sampling/importance_sampling_ratio/min": 0.6564763188362122, "sampling/sampling_logp_difference/max": 0.4208686351776123, "sampling/sampling_logp_difference/mean": 0.01750544086098671, "step": 130, "step_time": 27.429522439837456 }, { "clip_ratio/high_max": 0.0009249262996794035, "clip_ratio/high_mean": 0.0009249262996794035, "clip_ratio/low_mean": 0.000166456480656052, "clip_ratio/low_min": 0.000166456480656052, "clip_ratio/region_mean": 0.0010913827815481152, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1916.0, "completions/mean_length": 852.7333984375, "completions/mean_terminated_length": 694.867919921875, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.3944722463687261, "epoch": 0.19096209912536444, "frac_reward_zero_std": 0.0, "grad_norm": 0.0137899499386549, "learning_rate": 2e-05, "loss": 0.021337302401661873, "num_tokens": 7227849.0, "reward": 0.18362630903720856, "reward_std": 0.6749227046966553, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.41637369990348816, "rewards/length_penalty/std": 0.27475234866142273, "sampling/importance_sampling_ratio/max": 1.431358814239502, "sampling/importance_sampling_ratio/mean": 0.9860085248947144, "sampling/importance_sampling_ratio/min": 0.6481273174285889, "sampling/sampling_logp_difference/max": 0.4336681365966797, "sampling/sampling_logp_difference/mean": 0.023906156420707703, "step": 131, "step_time": 26.295353600988165 }, { "clip_ratio/high_max": 0.0013289499426415812, "clip_ratio/high_mean": 0.0013289499426415812, "clip_ratio/low_mean": 3.034164789520825e-05, "clip_ratio/low_min": 3.034164789520825e-05, "clip_ratio/region_mean": 0.0013592915784101933, "completions/clipped_ratio": 0.0, "completions/max_length": 1136.0, "completions/max_terminated_length": 1136.0, "completions/mean_length": 498.45001220703125, "completions/mean_terminated_length": 498.45001220703125, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.20351594934860864, "epoch": 0.1924198250728863, "frac_reward_zero_std": 0.0, "grad_norm": 0.009152635000646114, "learning_rate": 2e-05, "loss": -0.0025878362357616425, "num_tokens": 7264976.0, "reward": 0.6899495720863342, "reward_std": 0.289070725440979, "rewards/correctness/mean": 0.9333333373069763, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.24338379502296448, "rewards/length_penalty/std": 0.12275136262178421, "sampling/importance_sampling_ratio/max": 1.4464099407196045, "sampling/importance_sampling_ratio/mean": 0.9926159381866455, "sampling/importance_sampling_ratio/min": 0.6483365893363953, "sampling/sampling_logp_difference/max": 0.43334531784057617, "sampling/sampling_logp_difference/mean": 0.013126187026500702, "step": 132, "step_time": 15.227134235436097 }, { "clip_ratio/high_max": 0.0009851352369878441, "clip_ratio/high_mean": 0.0009851352369878441, "clip_ratio/low_mean": 6.253908698757489e-05, "clip_ratio/low_min": 6.253908698757489e-05, "clip_ratio/region_mean": 0.001047674323975419, "completions/clipped_ratio": 0.0, "completions/max_length": 1509.0, "completions/max_terminated_length": 1509.0, "completions/mean_length": 743.1000366210938, "completions/mean_terminated_length": 743.1000366210938, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.14064423739910126, "epoch": 0.19387755102040816, "frac_reward_zero_std": 0.0, "grad_norm": 0.011004043743014336, "learning_rate": 2e-05, "loss": -0.014399453997612, "num_tokens": 7314702.0, "reward": 0.40382489562034607, "reward_std": 0.4183441996574402, "rewards/correctness/mean": 0.7666666507720947, "rewards/correctness/std": 0.42652183771133423, "rewards/length_penalty/mean": -0.36284178495407104, "rewards/length_penalty/std": 0.18375426530838013, "sampling/importance_sampling_ratio/max": 1.448881983757019, "sampling/importance_sampling_ratio/mean": 0.9950322508811951, "sampling/importance_sampling_ratio/min": 0.6690908670425415, "sampling/sampling_logp_difference/max": 0.40183544158935547, "sampling/sampling_logp_difference/mean": 0.009434840641915798, "step": 133, "step_time": 19.896736015100032 }, { "clip_ratio/high_max": 0.0009553819812329797, "clip_ratio/high_mean": 0.0009553819812329797, "clip_ratio/low_mean": 0.00016222599030394727, "clip_ratio/low_min": 0.00016222599030394727, "clip_ratio/region_mean": 0.0011176079569850117, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1968.0, "completions/mean_length": 1053.283447265625, "completions/mean_terminated_length": 877.7451171875, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.3020109136899312, "epoch": 0.19533527696793002, "frac_reward_zero_std": 0.0, "grad_norm": 0.01610954850912094, "learning_rate": 2e-05, "loss": 0.03780611976981163, "num_tokens": 7385539.0, "reward": 0.0523681677877903, "reward_std": 0.7675083875656128, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.5142984986305237, "rewards/length_penalty/std": 0.32769298553466797, "sampling/importance_sampling_ratio/max": 1.450823187828064, "sampling/importance_sampling_ratio/mean": 0.9897510409355164, "sampling/importance_sampling_ratio/min": 0.3544185757637024, "sampling/sampling_logp_difference/max": 1.0372766256332397, "sampling/sampling_logp_difference/mean": 0.017968930304050446, "step": 134, "step_time": 28.227011064067483 }, { "clip_ratio/high_max": 0.000593315263055653, "clip_ratio/high_mean": 0.000593315263055653, "clip_ratio/low_mean": 0.0001034647757478524, "clip_ratio/low_min": 0.0001034647757478524, "clip_ratio/region_mean": 0.0006967800363781862, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 796.8833618164062, "completions/mean_terminated_length": 631.6415405273438, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.3218641405304273, "epoch": 0.1967930029154519, "frac_reward_zero_std": 0.0, "grad_norm": 0.011188547126948833, "learning_rate": 2e-05, "loss": 0.026850301772356033, "num_tokens": 7436682.0, "reward": 0.2108968198299408, "reward_std": 0.7179157137870789, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.3891032040119171, "rewards/length_penalty/std": 0.33137357234954834, "sampling/importance_sampling_ratio/max": 1.3966896533966064, "sampling/importance_sampling_ratio/mean": 0.989035427570343, "sampling/importance_sampling_ratio/min": 0.6542773842811584, "sampling/sampling_logp_difference/max": 0.4242238998413086, "sampling/sampling_logp_difference/mean": 0.019122382625937462, "step": 135, "step_time": 25.719649199862033 }, { "clip_ratio/high_max": 0.0011213517136638984, "clip_ratio/high_mean": 0.0011213517136638984, "clip_ratio/low_mean": 8.593292295699939e-05, "clip_ratio/low_min": 8.593292295699939e-05, "clip_ratio/region_mean": 0.0012072846293449402, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1966.0, "completions/mean_length": 651.3333740234375, "completions/mean_terminated_length": 603.1724243164062, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.42364654938379925, "epoch": 0.19825072886297376, "frac_reward_zero_std": 0.0, "grad_norm": 0.014928486198186874, "learning_rate": 2e-05, "loss": -0.004173014312982559, "num_tokens": 7481172.0, "reward": 0.4319661557674408, "reward_std": 0.5589545369148254, "rewards/correctness/mean": 0.75, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.3180338442325592, "rewards/length_penalty/std": 0.219534769654274, "sampling/importance_sampling_ratio/max": 1.714572787284851, "sampling/importance_sampling_ratio/mean": 0.985823392868042, "sampling/importance_sampling_ratio/min": 0.6611449718475342, "sampling/sampling_logp_difference/max": 0.5391639471054077, "sampling/sampling_logp_difference/mean": 0.024284308776259422, "step": 136, "step_time": 26.223453251179308 }, { "clip_ratio/high_max": 0.001284141922951676, "clip_ratio/high_mean": 0.001284141922951676, "clip_ratio/low_mean": 2.4341560977821548e-05, "clip_ratio/low_min": 2.4341560977821548e-05, "clip_ratio/region_mean": 0.0013084834742282208, "completions/clipped_ratio": 0.0, "completions/max_length": 1802.0, "completions/max_terminated_length": 1802.0, "completions/mean_length": 617.5333862304688, "completions/mean_terminated_length": 617.5333862304688, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.23057991514603296, "epoch": 0.19970845481049562, "frac_reward_zero_std": 0.0, "grad_norm": 0.011205561459064484, "learning_rate": 2e-05, "loss": -0.0336463563144207, "num_tokens": 7522004.0, "reward": 0.48180341720581055, "reward_std": 0.49382373690605164, "rewards/correctness/mean": 0.7833333611488342, "rewards/correctness/std": 0.41545021533966064, "rewards/length_penalty/mean": -0.3015299439430237, "rewards/length_penalty/std": 0.15826867520809174, "sampling/importance_sampling_ratio/max": 1.5545979738235474, "sampling/importance_sampling_ratio/mean": 0.9920415282249451, "sampling/importance_sampling_ratio/min": 0.6387434601783752, "sampling/sampling_logp_difference/max": 0.44825243949890137, "sampling/sampling_logp_difference/mean": 0.014765698462724686, "step": 137, "step_time": 21.748775377636775 }, { "clip_ratio/high_max": 0.0005354360716106991, "clip_ratio/high_mean": 0.0005354360716106991, "clip_ratio/low_mean": 2.5948414986487478e-05, "clip_ratio/low_min": 2.5948414986487478e-05, "clip_ratio/region_mean": 0.0005613844841718674, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 647.5000610351562, "completions/mean_terminated_length": 623.7626953125, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.21775240947802862, "epoch": 0.20116618075801748, "frac_reward_zero_std": 0.0, "grad_norm": 0.010806845501065254, "learning_rate": 2e-05, "loss": -0.007559105753898621, "num_tokens": 7564524.0, "reward": 0.30050456523895264, "reward_std": 0.5465244650840759, "rewards/correctness/mean": 0.6166666746139526, "rewards/correctness/std": 0.4903014302253723, "rewards/length_penalty/mean": -0.316162109375, "rewards/length_penalty/std": 0.1294468343257904, "sampling/importance_sampling_ratio/max": 1.4604040384292603, "sampling/importance_sampling_ratio/mean": 0.9926302433013916, "sampling/importance_sampling_ratio/min": 0.5560197234153748, "sampling/sampling_logp_difference/max": 0.586951494216919, "sampling/sampling_logp_difference/mean": 0.014007749035954475, "step": 138, "step_time": 24.29082035413012 }, { "clip_ratio/high_max": 0.0009670745542583367, "clip_ratio/high_mean": 0.0009670745542583367, "clip_ratio/low_mean": 0.000139080656178218, "clip_ratio/low_min": 0.000139080656178218, "clip_ratio/region_mean": 0.001106155231051768, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1682.0, "completions/mean_length": 747.7833862304688, "completions/mean_terminated_length": 679.3508911132812, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.23754004389047623, "epoch": 0.20262390670553937, "frac_reward_zero_std": 0.0, "grad_norm": 0.01702095754444599, "learning_rate": 2e-05, "loss": 0.018035396933555603, "num_tokens": 7613141.0, "reward": 0.33487144112586975, "reward_std": 0.5780368447303772, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.3651285767555237, "rewards/length_penalty/std": 0.2624134123325348, "sampling/importance_sampling_ratio/max": 1.4660247564315796, "sampling/importance_sampling_ratio/mean": 0.991218090057373, "sampling/importance_sampling_ratio/min": 0.6518484354019165, "sampling/sampling_logp_difference/max": 0.42794322967529297, "sampling/sampling_logp_difference/mean": 0.015252550132572651, "step": 139, "step_time": 25.713777780067176 }, { "clip_ratio/high_max": 0.001041515274361397, "clip_ratio/high_mean": 0.001041515274361397, "clip_ratio/low_mean": 8.830907124017055e-05, "clip_ratio/low_min": 8.830907124017055e-05, "clip_ratio/region_mean": 0.0011298243383256097, "completions/clipped_ratio": 0.0, "completions/max_length": 1405.0, "completions/max_terminated_length": 1405.0, "completions/mean_length": 550.9666748046875, "completions/mean_terminated_length": 550.9666748046875, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.20439065992832184, "epoch": 0.20408163265306123, "frac_reward_zero_std": 0.0, "grad_norm": 0.010910374112427235, "learning_rate": 2e-05, "loss": -0.002087140455842018, "num_tokens": 7650519.0, "reward": 0.6976400017738342, "reward_std": 0.2075270414352417, "rewards/correctness/mean": 0.9666666388511658, "rewards/correctness/std": 0.1810203343629837, "rewards/length_penalty/mean": -0.2690266966819763, "rewards/length_penalty/std": 0.11686760932207108, "sampling/importance_sampling_ratio/max": 1.4039456844329834, "sampling/importance_sampling_ratio/mean": 0.9930278062820435, "sampling/importance_sampling_ratio/min": 0.6248584389686584, "sampling/sampling_logp_difference/max": 0.4702301025390625, "sampling/sampling_logp_difference/mean": 0.013711715117096901, "step": 140, "step_time": 17.20439298520796 }, { "clip_ratio/high_max": 0.00038998989960722, "clip_ratio/high_mean": 0.00038998989960722, "clip_ratio/low_mean": 0.000349673655970643, "clip_ratio/low_min": 0.000349673655970643, "clip_ratio/region_mean": 0.0007396635483019054, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1954.0, "completions/mean_length": 926.4667358398438, "completions/mean_terminated_length": 728.549072265625, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.36274297535419464, "epoch": 0.2055393586005831, "frac_reward_zero_std": 0.0, "grad_norm": 0.01296077761799097, "learning_rate": 2e-05, "loss": 0.06322832405567169, "num_tokens": 7710807.0, "reward": 0.014290365390479565, "reward_std": 0.7337704300880432, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.4523763060569763, "rewards/length_penalty/std": 0.3023598790168762, "sampling/importance_sampling_ratio/max": 1.8982439041137695, "sampling/importance_sampling_ratio/mean": 0.9872742891311646, "sampling/importance_sampling_ratio/min": 0.5476210713386536, "sampling/sampling_logp_difference/max": 0.6409292221069336, "sampling/sampling_logp_difference/mean": 0.02172762341797352, "step": 141, "step_time": 26.343239391921088 }, { "clip_ratio/high_max": 0.0005487072364000293, "clip_ratio/high_mean": 0.0005487072364000293, "clip_ratio/low_mean": 0.0002699390655228247, "clip_ratio/low_min": 0.0002699390655228247, "clip_ratio/region_mean": 0.0008186462994975349, "completions/clipped_ratio": 0.3500000238418579, "completions/max_length": 2048.0, "completions/max_terminated_length": 1864.0, "completions/mean_length": 1286.5001220703125, "completions/mean_terminated_length": 876.4615478515625, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.3190828462441762, "epoch": 0.20699708454810495, "frac_reward_zero_std": 0.0, "grad_norm": 0.019596340134739876, "learning_rate": 2e-05, "loss": 0.07146541029214859, "num_tokens": 7794597.0, "reward": -0.34484052658081055, "reward_std": 0.5870059728622437, "rewards/correctness/mean": 0.28333333134651184, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.628173828125, "rewards/length_penalty/std": 0.32059621810913086, "sampling/importance_sampling_ratio/max": 2.1116273403167725, "sampling/importance_sampling_ratio/mean": 0.9887951016426086, "sampling/importance_sampling_ratio/min": 0.30148351192474365, "sampling/sampling_logp_difference/max": 1.1990399360656738, "sampling/sampling_logp_difference/mean": 0.019383693113923073, "step": 142, "step_time": 28.993721147766337 }, { "clip_ratio/high_max": 0.0006226475985992389, "clip_ratio/high_mean": 0.0006226475985992389, "clip_ratio/low_mean": 9.212004079017788e-05, "clip_ratio/low_min": 9.212004079017788e-05, "clip_ratio/region_mean": 0.0007147676418147361, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 2007.0, "completions/mean_length": 1030.4833984375, "completions/mean_terminated_length": 937.9818115234375, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "entropy": 0.32883496085802716, "epoch": 0.20845481049562684, "frac_reward_zero_std": 0.0, "grad_norm": 0.01710645481944084, "learning_rate": 2e-05, "loss": 0.01035105250775814, "num_tokens": 7861916.0, "reward": -0.3031657040119171, "reward_std": 0.4475523829460144, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.5031656622886658, "rewards/length_penalty/std": 0.25336435437202454, "sampling/importance_sampling_ratio/max": 1.4255783557891846, "sampling/importance_sampling_ratio/mean": 0.9885072112083435, "sampling/importance_sampling_ratio/min": 0.5910841822624207, "sampling/sampling_logp_difference/max": 0.5257967710494995, "sampling/sampling_logp_difference/mean": 0.01998257078230381, "step": 143, "step_time": 26.875178738264367 }, { "clip_ratio/high_max": 0.0009803994495693284, "clip_ratio/high_mean": 0.0009803994495693284, "clip_ratio/low_mean": 0.00010792129614856094, "clip_ratio/low_min": 0.00010792129614856094, "clip_ratio/region_mean": 0.001088320719039378, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1951.0, "completions/mean_length": 718.800048828125, "completions/mean_terminated_length": 571.1111450195312, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.3294060428937276, "epoch": 0.2099125364431487, "frac_reward_zero_std": 0.0, "grad_norm": 0.008123371750116348, "learning_rate": 2e-05, "loss": 0.004980289377272129, "num_tokens": 7909194.0, "reward": 0.44902345538139343, "reward_std": 0.6170951724052429, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.3509765565395355, "rewards/length_penalty/std": 0.25817549228668213, "sampling/importance_sampling_ratio/max": 1.4503473043441772, "sampling/importance_sampling_ratio/mean": 0.9886658191680908, "sampling/importance_sampling_ratio/min": 0.6502314805984497, "sampling/sampling_logp_difference/max": 0.43042683601379395, "sampling/sampling_logp_difference/mean": 0.019626792520284653, "step": 144, "step_time": 25.531999272527173 }, { "clip_ratio/high_max": 0.0009709181807314357, "clip_ratio/high_mean": 0.0009709181807314357, "clip_ratio/low_mean": 0.00015339334883416692, "clip_ratio/low_min": 0.00015339334883416692, "clip_ratio/region_mean": 0.0011243115295656025, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1719.0, "completions/mean_length": 876.11669921875, "completions/mean_terminated_length": 792.4107666015625, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.2737928008039792, "epoch": 0.21137026239067055, "frac_reward_zero_std": 0.0, "grad_norm": 0.013971668668091297, "learning_rate": 2e-05, "loss": 0.05236930772662163, "num_tokens": 7966161.0, "reward": 0.2388753443956375, "reward_std": 0.6455968022346497, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.4277913272380829, "rewards/length_penalty/std": 0.2562803328037262, "sampling/importance_sampling_ratio/max": 1.402762532234192, "sampling/importance_sampling_ratio/mean": 0.9905370473861694, "sampling/importance_sampling_ratio/min": 0.4719122052192688, "sampling/sampling_logp_difference/max": 0.7509623765945435, "sampling/sampling_logp_difference/mean": 0.01711660996079445, "step": 145, "step_time": 26.461914269486442 }, { "clip_ratio/high_max": 0.0006462544382278187, "clip_ratio/high_mean": 0.0006462544382278187, "clip_ratio/low_mean": 0.0002863618453072074, "clip_ratio/low_min": 0.0002863618453072074, "clip_ratio/region_mean": 0.0009326162786843876, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1960.0, "completions/mean_length": 709.1666870117188, "completions/mean_terminated_length": 503.19232177734375, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.40346813201904297, "epoch": 0.21282798833819241, "frac_reward_zero_std": 0.0, "grad_norm": 0.009171973913908005, "learning_rate": 2e-05, "loss": 0.0015175435692071915, "num_tokens": 8012971.0, "reward": 0.20372723042964935, "reward_std": 0.679226815700531, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.3462727963924408, "rewards/length_penalty/std": 0.2919903099536896, "sampling/importance_sampling_ratio/max": 1.4315117597579956, "sampling/importance_sampling_ratio/mean": 0.9860925078392029, "sampling/importance_sampling_ratio/min": 0.6461740136146545, "sampling/sampling_logp_difference/max": 0.43668651580810547, "sampling/sampling_logp_difference/mean": 0.02341708354651928, "step": 146, "step_time": 25.89526958204806 }, { "clip_ratio/high_max": 0.000788814082625322, "clip_ratio/high_mean": 0.000788814082625322, "clip_ratio/low_mean": 7.270114535155396e-05, "clip_ratio/low_min": 7.270114535155396e-05, "clip_ratio/region_mean": 0.000861515227976876, "completions/clipped_ratio": 0.0, "completions/max_length": 1075.0, "completions/max_terminated_length": 1075.0, "completions/mean_length": 495.5666809082031, "completions/mean_terminated_length": 495.5666809082031, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.18527658035357794, "epoch": 0.21428571428571427, "frac_reward_zero_std": 0.0, "grad_norm": 0.010867736302316189, "learning_rate": 2e-05, "loss": 0.004002865869551897, "num_tokens": 8046455.0, "reward": 0.42469078302383423, "reward_std": 0.5008008480072021, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.24197591841220856, "rewards/length_penalty/std": 0.13876907527446747, "sampling/importance_sampling_ratio/max": 1.413368821144104, "sampling/importance_sampling_ratio/mean": 0.9933928847312927, "sampling/importance_sampling_ratio/min": 0.6621342301368713, "sampling/sampling_logp_difference/max": 0.41228699684143066, "sampling/sampling_logp_difference/mean": 0.012012646533548832, "step": 147, "step_time": 13.71861265343614 }, { "clip_ratio/high_max": 0.0009729232527509643, "clip_ratio/high_mean": 0.0009729232527509643, "clip_ratio/low_mean": 3.495525743346661e-05, "clip_ratio/low_min": 3.495525743346661e-05, "clip_ratio/region_mean": 0.0010078785150350693, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1328.0, "completions/mean_length": 769.8167114257812, "completions/mean_terminated_length": 702.5438842773438, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.2404237985610962, "epoch": 0.21574344023323616, "frac_reward_zero_std": 0.0, "grad_norm": 0.028219083324074745, "learning_rate": 2e-05, "loss": 0.11528102308511734, "num_tokens": 8096494.0, "reward": 0.40744632482528687, "reward_std": 0.5218658447265625, "rewards/correctness/mean": 0.7833333611488342, "rewards/correctness/std": 0.41545024514198303, "rewards/length_penalty/mean": -0.37588703632354736, "rewards/length_penalty/std": 0.19455382227897644, "sampling/importance_sampling_ratio/max": 1.4559873342514038, "sampling/importance_sampling_ratio/mean": 0.9913713335990906, "sampling/importance_sampling_ratio/min": 0.6405834555625916, "sampling/sampling_logp_difference/max": 0.445375919342041, "sampling/sampling_logp_difference/mean": 0.014926444739103317, "step": 148, "step_time": 25.856907369336113 }, { "clip_ratio/high_max": 0.0011667126091197133, "clip_ratio/high_mean": 0.0011667126091197133, "clip_ratio/low_mean": 0.00011985406066135813, "clip_ratio/low_min": 0.00011985406066135813, "clip_ratio/region_mean": 0.001286566713436817, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1829.0, "completions/mean_length": 788.2667236328125, "completions/mean_terminated_length": 744.8275756835938, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.28645333151022595, "epoch": 0.21720116618075802, "frac_reward_zero_std": 0.0, "grad_norm": 0.012622080743312836, "learning_rate": 2e-05, "loss": 0.03589104115962982, "num_tokens": 8149690.0, "reward": 0.16510418057441711, "reward_std": 0.5626128911972046, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.38489583134651184, "rewards/length_penalty/std": 0.20437301695346832, "sampling/importance_sampling_ratio/max": 1.4403685331344604, "sampling/importance_sampling_ratio/mean": 0.9900524616241455, "sampling/importance_sampling_ratio/min": 0.5904785990715027, "sampling/sampling_logp_difference/max": 0.5268218517303467, "sampling/sampling_logp_difference/mean": 0.017937393859028816, "step": 149, "step_time": 26.160948349162936 }, { "clip_ratio/high_max": 0.0005951016404045125, "clip_ratio/high_mean": 0.0005951016404045125, "clip_ratio/low_mean": 0.0001770943466302318, "clip_ratio/low_min": 0.0001770943466302318, "clip_ratio/region_mean": 0.0007721959870347442, "completions/clipped_ratio": 0.21666668355464935, "completions/max_length": 2048.0, "completions/max_terminated_length": 1881.0, "completions/mean_length": 930.1000366210938, "completions/mean_terminated_length": 620.8936157226562, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.3673850744962692, "epoch": 0.21865889212827988, "frac_reward_zero_std": 0.0, "grad_norm": 0.015178069472312927, "learning_rate": 2e-05, "loss": -0.0044382475316524506, "num_tokens": 8210576.0, "reward": -0.20415040850639343, "reward_std": 0.6173945665359497, "rewards/correctness/mean": 0.25, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.45415037870407104, "rewards/length_penalty/std": 0.3532169461250305, "sampling/importance_sampling_ratio/max": 1.442487120628357, "sampling/importance_sampling_ratio/mean": 0.9874463677406311, "sampling/importance_sampling_ratio/min": 0.5390064716339111, "sampling/sampling_logp_difference/max": 0.6180276870727539, "sampling/sampling_logp_difference/mean": 0.02184767834842205, "step": 150, "step_time": 26.75312972185202 }, { "clip_ratio/high_max": 0.0007837310889347767, "clip_ratio/high_mean": 0.0007837310889347767, "clip_ratio/low_mean": 0.00012820728685862073, "clip_ratio/low_min": 0.00012820728685862073, "clip_ratio/region_mean": 0.0009119383757933974, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1938.0, "completions/mean_length": 836.0333862304688, "completions/mean_terminated_length": 749.4642944335938, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.23776968320210776, "epoch": 0.22011661807580174, "frac_reward_zero_std": 0.0, "grad_norm": 0.011509308591485023, "learning_rate": 2e-05, "loss": -0.003738299012184143, "num_tokens": 8264918.0, "reward": 0.12511393427848816, "reward_std": 0.6106396913528442, "rewards/correctness/mean": 0.5333333611488342, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.4082193970680237, "rewards/length_penalty/std": 0.2634795904159546, "sampling/importance_sampling_ratio/max": 1.413131594657898, "sampling/importance_sampling_ratio/mean": 0.9915941953659058, "sampling/importance_sampling_ratio/min": 0.6075857281684875, "sampling/sampling_logp_difference/max": 0.49826204776763916, "sampling/sampling_logp_difference/mean": 0.014894966036081314, "step": 151, "step_time": 26.18424436938949 }, { "clip_ratio/high_max": 0.0011510254771565087, "clip_ratio/high_mean": 0.0011510254771565087, "clip_ratio/low_mean": 0.0002647102919581812, "clip_ratio/low_min": 0.0002647102919581812, "clip_ratio/region_mean": 0.0014157357703273494, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1879.0, "completions/mean_length": 746.4500122070312, "completions/mean_terminated_length": 628.1272583007812, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.4303782532612483, "epoch": 0.22157434402332363, "frac_reward_zero_std": 0.0, "grad_norm": 0.015101080760359764, "learning_rate": 2e-05, "loss": 0.006294237449765205, "num_tokens": 8314415.0, "reward": 0.23552247881889343, "reward_std": 0.6499772667884827, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.3644775450229645, "rewards/length_penalty/std": 0.24947568774223328, "sampling/importance_sampling_ratio/max": 1.4334946870803833, "sampling/importance_sampling_ratio/mean": 0.9842382669448853, "sampling/importance_sampling_ratio/min": 0.6101853251457214, "sampling/sampling_logp_difference/max": 0.49399256706237793, "sampling/sampling_logp_difference/mean": 0.026771733537316322, "step": 152, "step_time": 25.680529301287606 }, { "clip_ratio/high_max": 0.0008009954132527733, "clip_ratio/high_mean": 0.0008009954132527733, "clip_ratio/low_mean": 7.452095572565061e-05, "clip_ratio/low_min": 7.452095572565061e-05, "clip_ratio/region_mean": 0.0008755163774670413, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1615.0, "completions/mean_length": 868.9166870117188, "completions/mean_terminated_length": 806.8596801757812, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.23239794125159582, "epoch": 0.2230320699708455, "frac_reward_zero_std": 0.0, "grad_norm": 0.0174455214291811, "learning_rate": 2e-05, "loss": 0.07516114413738251, "num_tokens": 8372510.0, "reward": 0.1590576320886612, "reward_std": 0.6134790182113647, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.4242757260799408, "rewards/length_penalty/std": 0.1961939036846161, "sampling/importance_sampling_ratio/max": 1.432678461074829, "sampling/importance_sampling_ratio/mean": 0.9919102787971497, "sampling/importance_sampling_ratio/min": 0.6673089265823364, "sampling/sampling_logp_difference/max": 0.40450215339660645, "sampling/sampling_logp_difference/mean": 0.014779479242861271, "step": 153, "step_time": 26.61921369191259 }, { "clip_ratio/high_max": 0.0013186451833462343, "clip_ratio/high_mean": 0.0013186451833462343, "clip_ratio/low_mean": 1.9716865305478375e-05, "clip_ratio/low_min": 1.9716865305478375e-05, "clip_ratio/region_mean": 0.0013383620486517127, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1193.0, "completions/mean_length": 632.5167236328125, "completions/mean_terminated_length": 608.5254516601562, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.19110674659411112, "epoch": 0.22448979591836735, "frac_reward_zero_std": 0.0, "grad_norm": 0.011251011863350868, "learning_rate": 2e-05, "loss": 0.006448462605476379, "num_tokens": 8414591.0, "reward": 0.4911539852619171, "reward_std": 0.46878206729888916, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.30884602665901184, "rewards/length_penalty/std": 0.15812522172927856, "sampling/importance_sampling_ratio/max": 1.4671045541763306, "sampling/importance_sampling_ratio/mean": 0.9933434724807739, "sampling/importance_sampling_ratio/min": 0.6210270524024963, "sampling/sampling_logp_difference/max": 0.4763805866241455, "sampling/sampling_logp_difference/mean": 0.01292010210454464, "step": 154, "step_time": 24.742094105109572 }, { "clip_ratio/high_max": 0.0012121789429026346, "clip_ratio/high_mean": 0.0012121789429026346, "clip_ratio/low_mean": 0.0001811962744492727, "clip_ratio/low_min": 0.0001811962744492727, "clip_ratio/region_mean": 0.0013933752197772264, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1872.0, "completions/mean_length": 729.61669921875, "completions/mean_terminated_length": 707.2711791992188, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.2897387221455574, "epoch": 0.2259475218658892, "frac_reward_zero_std": 0.0, "grad_norm": 0.016924167051911354, "learning_rate": 2e-05, "loss": 0.0368056520819664, "num_tokens": 8463068.0, "reward": 0.32707521319389343, "reward_std": 0.5871769189834595, "rewards/correctness/mean": 0.6833333373069763, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.3562581241130829, "rewards/length_penalty/std": 0.23875026404857635, "sampling/importance_sampling_ratio/max": 1.4597691297531128, "sampling/importance_sampling_ratio/mean": 0.9895320534706116, "sampling/importance_sampling_ratio/min": 0.40709778666496277, "sampling/sampling_logp_difference/max": 0.8987019062042236, "sampling/sampling_logp_difference/mean": 0.019146829843521118, "step": 155, "step_time": 25.909327471861616 }, { "clip_ratio/high_max": 0.0007107193402286308, "clip_ratio/high_mean": 0.0007107193402286308, "clip_ratio/low_mean": 0.00023871100953935334, "clip_ratio/low_min": 0.00023871100953935334, "clip_ratio/region_mean": 0.0009494303424920266, "completions/clipped_ratio": 0.11666667461395264, "completions/max_length": 2048.0, "completions/max_terminated_length": 1929.0, "completions/mean_length": 794.4500122070312, "completions/mean_terminated_length": 628.8867797851562, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 0.3392493550976117, "epoch": 0.22740524781341107, "frac_reward_zero_std": 0.0, "grad_norm": 0.009066332131624222, "learning_rate": 2e-05, "loss": -0.0009449312929064035, "num_tokens": 8514985.0, "reward": 0.2620849609375, "reward_std": 0.7496296763420105, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.3879150450229645, "rewards/length_penalty/std": 0.2956892251968384, "sampling/importance_sampling_ratio/max": 1.6749331951141357, "sampling/importance_sampling_ratio/mean": 0.9876399636268616, "sampling/importance_sampling_ratio/min": 0.6394190192222595, "sampling/sampling_logp_difference/max": 0.5157732963562012, "sampling/sampling_logp_difference/mean": 0.021354639902710915, "step": 156, "step_time": 25.676308020716533 }, { "clip_ratio/high_max": 0.0007681013084948063, "clip_ratio/high_mean": 0.0007681013084948063, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007681013084948063, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1442.0, "completions/mean_length": 697.9833984375, "completions/mean_terminated_length": 651.4310302734375, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.17722446843981743, "epoch": 0.22886297376093295, "frac_reward_zero_std": 0.0, "grad_norm": 0.016081683337688446, "learning_rate": 2e-05, "loss": 0.056921664625406265, "num_tokens": 8561214.0, "reward": 0.5925211906433105, "reward_std": 0.4066368341445923, "rewards/correctness/mean": 0.9333333373069763, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.34081217646598816, "rewards/length_penalty/std": 0.20141668617725372, "sampling/importance_sampling_ratio/max": 1.4456247091293335, "sampling/importance_sampling_ratio/mean": 0.993467390537262, "sampling/importance_sampling_ratio/min": 0.6672379970550537, "sampling/sampling_logp_difference/max": 0.40460848808288574, "sampling/sampling_logp_difference/mean": 0.012250185012817383, "step": 157, "step_time": 25.352831785101444 }, { "clip_ratio/high_max": 0.0002853689814704315, "clip_ratio/high_mean": 0.0002853689814704315, "clip_ratio/low_mean": 0.00025238765495790477, "clip_ratio/low_min": 0.00025238765495790477, "clip_ratio/region_mean": 0.0005377566509802515, "completions/clipped_ratio": 0.40000003576278687, "completions/max_length": 2048.0, "completions/max_terminated_length": 2012.0, "completions/mean_length": 1259.4666748046875, "completions/mean_terminated_length": 733.7777709960938, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.35649363199869794, "epoch": 0.2303206997084548, "frac_reward_zero_std": 0.0, "grad_norm": 0.020887818187475204, "learning_rate": 2e-05, "loss": 0.034592047333717346, "num_tokens": 8641882.0, "reward": -0.03164062649011612, "reward_std": 0.8320719599723816, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.6149739623069763, "rewards/length_penalty/std": 0.35453444719314575, "sampling/importance_sampling_ratio/max": 1.694777011871338, "sampling/importance_sampling_ratio/mean": 0.9877781867980957, "sampling/importance_sampling_ratio/min": 0.5174592733383179, "sampling/sampling_logp_difference/max": 0.6588244438171387, "sampling/sampling_logp_difference/mean": 0.021618111059069633, "step": 158, "step_time": 28.766278768889606 }, { "clip_ratio/high_max": 0.0012234286599171658, "clip_ratio/high_mean": 0.0012234286599171658, "clip_ratio/low_mean": 0.00019811623981998613, "clip_ratio/low_min": 0.00019811623981998613, "clip_ratio/region_mean": 0.0014215449142890673, "completions/clipped_ratio": 0.06666667014360428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1571.0, "completions/mean_length": 800.9166870117188, "completions/mean_terminated_length": 711.8392944335938, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.2685914561152458, "epoch": 0.23177842565597667, "frac_reward_zero_std": 0.0, "grad_norm": 0.015032554976642132, "learning_rate": 2e-05, "loss": 0.04539729654788971, "num_tokens": 8695537.0, "reward": 0.39226076006889343, "reward_std": 0.5275649428367615, "rewards/correctness/mean": 0.7833333611488342, "rewards/correctness/std": 0.41545024514198303, "rewards/length_penalty/mean": -0.3910726010799408, "rewards/length_penalty/std": 0.22598691284656525, "sampling/importance_sampling_ratio/max": 1.5921074151992798, "sampling/importance_sampling_ratio/mean": 0.9907950758934021, "sampling/importance_sampling_ratio/min": 0.6347799897193909, "sampling/sampling_logp_difference/max": 0.4650585651397705, "sampling/sampling_logp_difference/mean": 0.016757389530539513, "step": 159, "step_time": 25.799214388476685 }, { "clip_ratio/high_max": 0.0013589256996056065, "clip_ratio/high_mean": 0.0013589256996056065, "clip_ratio/low_mean": 0.00021470187493832782, "clip_ratio/low_min": 0.00021470187493832782, "clip_ratio/region_mean": 0.001573627606073084, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 1981.0, "completions/mean_length": 800.1000366210938, "completions/mean_terminated_length": 686.654541015625, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 0.3904004245996475, "epoch": 0.23323615160349853, "frac_reward_zero_std": 0.0, "grad_norm": 0.019178809598088264, "learning_rate": 2e-05, "loss": 0.042169515043497086, "num_tokens": 8747233.0, "reward": 0.009326172061264515, "reward_std": 0.6670224070549011, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49403220415115356, "rewards/length_penalty/mean": -0.39067381620407104, "rewards/length_penalty/std": 0.2804904282093048, "sampling/importance_sampling_ratio/max": 1.467087745666504, "sampling/importance_sampling_ratio/mean": 0.9868454337120056, "sampling/importance_sampling_ratio/min": 0.5541986227035522, "sampling/sampling_logp_difference/max": 0.5902321338653564, "sampling/sampling_logp_difference/mean": 0.02422356605529785, "step": 160, "step_time": 25.9806922392454 }, { "clip_ratio/high_max": 0.0010678241790931982, "clip_ratio/high_mean": 0.0010678241790931982, "clip_ratio/low_mean": 8.358072348831531e-05, "clip_ratio/low_min": 8.358072348831531e-05, "clip_ratio/region_mean": 0.0011514048965182155, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 761.3333740234375, "completions/mean_terminated_length": 693.614013671875, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.27356397608915967, "epoch": 0.23469387755102042, "frac_reward_zero_std": 0.0, "grad_norm": 0.023295298218727112, "learning_rate": 2e-05, "loss": 0.05298904329538345, "num_tokens": 8797223.0, "reward": 0.44492191076278687, "reward_std": 0.6138378381729126, "rewards/correctness/mean": 0.8166666626930237, "rewards/correctness/std": 0.39020493626594543, "rewards/length_penalty/mean": -0.3717447817325592, "rewards/length_penalty/std": 0.26436156034469604, "sampling/importance_sampling_ratio/max": 1.4474414587020874, "sampling/importance_sampling_ratio/mean": 0.9904724955558777, "sampling/importance_sampling_ratio/min": 0.6545323133468628, "sampling/sampling_logp_difference/max": 0.42383432388305664, "sampling/sampling_logp_difference/mean": 0.01725386641919613, "step": 161, "step_time": 25.533062344184145 }, { "clip_ratio/high_max": 0.0009161523485090584, "clip_ratio/high_mean": 0.0009161523485090584, "clip_ratio/low_mean": 0.00015165507769173323, "clip_ratio/low_min": 0.00015165507769173323, "clip_ratio/region_mean": 0.0010678074031602591, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1835.0, "completions/mean_length": 614.1666870117188, "completions/mean_terminated_length": 589.8643798828125, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.2376063788930575, "epoch": 0.23615160349854228, "frac_reward_zero_std": 0.0, "grad_norm": 0.015938283875584602, "learning_rate": 2e-05, "loss": 0.056312985718250275, "num_tokens": 8837513.0, "reward": 0.48344728350639343, "reward_std": 0.5652655363082886, "rewards/correctness/mean": 0.7833333611488342, "rewards/correctness/std": 0.41545021533966064, "rewards/length_penalty/mean": -0.2998860776424408, "rewards/length_penalty/std": 0.18964459002017975, "sampling/importance_sampling_ratio/max": 1.5897924900054932, "sampling/importance_sampling_ratio/mean": 0.9915858507156372, "sampling/importance_sampling_ratio/min": 0.6721543669700623, "sampling/sampling_logp_difference/max": 0.46360349655151367, "sampling/sampling_logp_difference/mean": 0.015679962933063507, "step": 162, "step_time": 24.472435717703775 }, { "clip_ratio/high_max": 0.0013966952683404088, "clip_ratio/high_mean": 0.0013966952683404088, "clip_ratio/low_mean": 2.8651783698781703e-05, "clip_ratio/low_min": 2.8651783698781703e-05, "clip_ratio/region_mean": 0.001425347038699935, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1896.0, "completions/mean_length": 1025.6500244140625, "completions/mean_terminated_length": 770.0625, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.2782551323374112, "epoch": 0.23760932944606414, "frac_reward_zero_std": 0.0, "grad_norm": 0.014002242125570774, "learning_rate": 2e-05, "loss": 0.022032344713807106, "num_tokens": 8907392.0, "reward": 0.08252767473459244, "reward_std": 0.7483339905738831, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.500805675983429, "rewards/length_penalty/std": 0.3090408444404602, "sampling/importance_sampling_ratio/max": 1.6037006378173828, "sampling/importance_sampling_ratio/mean": 0.9900583028793335, "sampling/importance_sampling_ratio/min": 0.6024936437606812, "sampling/sampling_logp_difference/max": 0.5066781044006348, "sampling/sampling_logp_difference/mean": 0.018255088478326797, "step": 163, "step_time": 28.169299229746684 }, { "clip_ratio/high_max": 0.0012512138249197353, "clip_ratio/high_mean": 0.0012512138249197353, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012512138249197353, "completions/clipped_ratio": 0.0, "completions/max_length": 560.0, "completions/max_terminated_length": 560.0, "completions/mean_length": 384.5166931152344, "completions/mean_terminated_length": 384.5166931152344, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.1906136597196261, "epoch": 0.239067055393586, "frac_reward_zero_std": 0.0, "grad_norm": 0.00936169270426035, "learning_rate": 2e-05, "loss": -0.0005168644711375237, "num_tokens": 8934753.0, "reward": 0.7122477293014526, "reward_std": 0.30102622509002686, "rewards/correctness/mean": 0.8999999761581421, "rewards/correctness/std": 0.3025316894054413, "rewards/length_penalty/mean": -0.18775227665901184, "rewards/length_penalty/std": 0.05292113497853279, "sampling/importance_sampling_ratio/max": 1.4079543352127075, "sampling/importance_sampling_ratio/mean": 0.9925669431686401, "sampling/importance_sampling_ratio/min": 0.6690069437026978, "sampling/sampling_logp_difference/max": 0.4019608497619629, "sampling/sampling_logp_difference/mean": 0.013436808250844479, "step": 164, "step_time": 7.938276870874688 }, { "clip_ratio/high_max": 0.0005091577768325806, "clip_ratio/high_mean": 0.0005091577768325806, "clip_ratio/low_mean": 5.713850987376645e-05, "clip_ratio/low_min": 5.713850987376645e-05, "clip_ratio/region_mean": 0.0005662962891316662, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 2022.0, "completions/mean_length": 882.8500366210938, "completions/mean_terminated_length": 677.2352905273438, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 0.301129013299942, "epoch": 0.24052478134110788, "frac_reward_zero_std": 0.0, "grad_norm": 0.011507508344948292, "learning_rate": 2e-05, "loss": 0.028687074780464172, "num_tokens": 8992344.0, "reward": 0.20225423574447632, "reward_std": 0.7268537282943726, "rewards/correctness/mean": 0.6333333253860474, "rewards/correctness/std": 0.48596110939979553, "rewards/length_penalty/mean": -0.43107908964157104, "rewards/length_penalty/std": 0.30351847410202026, "sampling/importance_sampling_ratio/max": 1.4683129787445068, "sampling/importance_sampling_ratio/mean": 0.9888500571250916, "sampling/importance_sampling_ratio/min": 0.6509833335876465, "sampling/sampling_logp_difference/max": 0.42927122116088867, "sampling/sampling_logp_difference/mean": 0.019634412601590157, "step": 165, "step_time": 26.11276886612177 }, { "clip_ratio/high_max": 0.0008709375009251138, "clip_ratio/high_mean": 0.0008709375009251138, "clip_ratio/low_mean": 0.00027229892536221695, "clip_ratio/low_min": 0.00027229892536221695, "clip_ratio/region_mean": 0.0011432364020341386, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 1950.0, "completions/mean_length": 935.9000244140625, "completions/mean_terminated_length": 657.875, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.4242909774184227, "epoch": 0.24198250728862974, "frac_reward_zero_std": 0.0, "grad_norm": 0.015190168283879757, "learning_rate": 2e-05, "loss": 0.015511132776737213, "num_tokens": 9051968.0, "reward": 0.29301759600639343, "reward_std": 0.7033945918083191, "rewards/correctness/mean": 0.75, "rewards/correctness/std": 0.436666876077652, "rewards/length_penalty/mean": -0.45698243379592896, "rewards/length_penalty/std": 0.30968427658081055, "sampling/importance_sampling_ratio/max": 1.5805938243865967, "sampling/importance_sampling_ratio/mean": 0.9843783378601074, "sampling/importance_sampling_ratio/min": 0.6149616241455078, "sampling/sampling_logp_difference/max": 0.48619544506073, "sampling/sampling_logp_difference/mean": 0.027219604700803757, "step": 166, "step_time": 26.639186922926456 }, { "clip_ratio/high_max": 0.0007828866728232242, "clip_ratio/high_mean": 0.0007828866728232242, "clip_ratio/low_mean": 3.87596907482172e-05, "clip_ratio/low_min": 3.87596907482172e-05, "clip_ratio/region_mean": 0.000821646358720803, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 425.5166931152344, "completions/mean_terminated_length": 425.5166931152344, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.1843807970484098, "epoch": 0.2434402332361516, "frac_reward_zero_std": 0.0, "grad_norm": 0.01017924677580595, "learning_rate": 2e-05, "loss": -0.004341884516179562, "num_tokens": 9080519.0, "reward": 0.7422282099723816, "reward_std": 0.21998131275177002, "rewards/correctness/mean": 0.949999988079071, "rewards/correctness/std": 0.2197841852903366, "rewards/length_penalty/mean": -0.20777180790901184, "rewards/length_penalty/std": 0.034709908068180084, "sampling/importance_sampling_ratio/max": 1.5466643571853638, "sampling/importance_sampling_ratio/mean": 0.9935150742530823, "sampling/importance_sampling_ratio/min": 0.6078754663467407, "sampling/sampling_logp_difference/max": 0.49778521060943604, "sampling/sampling_logp_difference/mean": 0.013069001026451588, "step": 167, "step_time": 9.47851428994909 }, { "clip_ratio/high_max": 0.0007199387158228395, "clip_ratio/high_mean": 0.0007199387158228395, "clip_ratio/low_mean": 0.00015790038863390995, "clip_ratio/low_min": 0.00015790038863390995, "clip_ratio/region_mean": 0.000877839093542813, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 2035.0, "completions/mean_length": 713.36669921875, "completions/mean_terminated_length": 592.0363159179688, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.28486377497514087, "epoch": 0.24489795918367346, "frac_reward_zero_std": 0.0, "grad_norm": 0.017850568518042564, "learning_rate": 2e-05, "loss": 0.0237590279430151, "num_tokens": 9128221.0, "reward": 0.45167645812034607, "reward_std": 0.6125007271766663, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4033755958080292, "rewards/length_penalty/mean": -0.3483235538005829, "rewards/length_penalty/std": 0.2630182206630707, "sampling/importance_sampling_ratio/max": 1.4393036365509033, "sampling/importance_sampling_ratio/mean": 0.9893081784248352, "sampling/importance_sampling_ratio/min": 0.6482274532318115, "sampling/sampling_logp_difference/max": 0.4335136413574219, "sampling/sampling_logp_difference/mean": 0.019400296732783318, "step": 168, "step_time": 25.452282634796575 }, { "clip_ratio/high_max": 0.000811036714973549, "clip_ratio/high_mean": 0.000811036714973549, "clip_ratio/low_mean": 0.0002101373320329003, "clip_ratio/low_min": 0.0002101373320329003, "clip_ratio/region_mean": 0.0010211740445811301, "completions/clipped_ratio": 0.1666666716337204, "completions/max_length": 2048.0, "completions/max_terminated_length": 2048.0, "completions/mean_length": 1134.5833740234375, "completions/mean_terminated_length": 951.8999633789062, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.38400957981745404, "epoch": 0.24635568513119532, "frac_reward_zero_std": 0.0, "grad_norm": 0.0206043291836977, "learning_rate": 2e-05, "loss": 0.0436839833855629, "num_tokens": 9202736.0, "reward": -0.15399578213691711, "reward_std": 0.6629047989845276, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.5539957880973816, "rewards/length_penalty/std": 0.30165985226631165, "sampling/importance_sampling_ratio/max": 1.6086947917938232, "sampling/importance_sampling_ratio/mean": 0.9861995577812195, "sampling/importance_sampling_ratio/min": 0.6074703335762024, "sampling/sampling_logp_difference/max": 0.49845194816589355, "sampling/sampling_logp_difference/mean": 0.024005858227610588, "step": 169, "step_time": 27.775138321798295 }, { "clip_ratio/high_max": 0.000899404549272731, "clip_ratio/high_mean": 0.000899404549272731, "clip_ratio/low_mean": 0.000148629956432463, "clip_ratio/low_min": 0.000148629956432463, "clip_ratio/region_mean": 0.0010480345032798748, "completions/clipped_ratio": 0.0, "completions/max_length": 1892.0, "completions/max_terminated_length": 1892.0, "completions/mean_length": 623.2166748046875, "completions/mean_terminated_length": 623.2166748046875, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.2634037708242734, "epoch": 0.2478134110787172, "frac_reward_zero_std": 0.0, "grad_norm": 0.010958128608763218, "learning_rate": 2e-05, "loss": -0.003399193286895752, "num_tokens": 9245929.0, "reward": 0.24569499492645264, "reward_std": 0.5535570383071899, "rewards/correctness/mean": 0.550000011920929, "rewards/correctness/std": 0.5016920566558838, "rewards/length_penalty/mean": -0.3043050169944763, "rewards/length_penalty/std": 0.15333330631256104, "sampling/importance_sampling_ratio/max": 1.6163140535354614, "sampling/importance_sampling_ratio/mean": 0.9908414483070374, "sampling/importance_sampling_ratio/min": 0.5295116305351257, "sampling/sampling_logp_difference/max": 0.6358001232147217, "sampling/sampling_logp_difference/mean": 0.017154477536678314, "step": 170, "step_time": 24.2347475157585 }, { "clip_ratio/high_max": 0.0006975887202618954, "clip_ratio/high_mean": 0.0006975887202618954, "clip_ratio/low_mean": 0.00013354327044604966, "clip_ratio/low_min": 0.00013354327044604966, "clip_ratio/region_mean": 0.0008311319931332642, "completions/clipped_ratio": 0.0, "completions/max_length": 859.0, "completions/max_terminated_length": 859.0, "completions/mean_length": 482.7500305175781, "completions/mean_terminated_length": 482.7500305175781, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.2617703974246979, "epoch": 0.24927113702623907, "frac_reward_zero_std": 0.0, "grad_norm": 0.013744283467531204, "learning_rate": 2e-05, "loss": 0.03806378319859505, "num_tokens": 9279434.0, "reward": 0.5309489369392395, "reward_std": 0.45812374353408813, "rewards/correctness/mean": 0.7666666507720947, "rewards/correctness/std": 0.42652183771133423, "rewards/length_penalty/mean": -0.2357177734375, "rewards/length_penalty/std": 0.09047216922044754, "sampling/importance_sampling_ratio/max": 1.6830754280090332, "sampling/importance_sampling_ratio/mean": 0.990826427936554, "sampling/importance_sampling_ratio/min": 0.6258879899978638, "sampling/sampling_logp_difference/max": 0.520622730255127, "sampling/sampling_logp_difference/mean": 0.0171992015093565, "step": 171, "step_time": 11.817887012613937 }, { "clip_ratio/high_max": 0.0007639260002179071, "clip_ratio/high_mean": 0.0007639260002179071, "clip_ratio/low_mean": 3.643783808608229e-05, "clip_ratio/low_min": 3.643783808608229e-05, "clip_ratio/region_mean": 0.0008003638358786702, "completions/clipped_ratio": 0.0, "completions/max_length": 859.0, "completions/max_terminated_length": 859.0, "completions/mean_length": 418.4000244140625, "completions/mean_terminated_length": 418.4000244140625, "completions/min_length": 103.0, "completions/min_terminated_length": 103.0, "entropy": 0.20873272667328516, "epoch": 0.25072886297376096, "frac_reward_zero_std": 0.0, "grad_norm": 0.010405924171209335, "learning_rate": 2e-05, "loss": 0.010939408093690872, "num_tokens": 9308038.0, "reward": 0.7623698115348816, "reward_std": 0.21447253227233887, "rewards/correctness/mean": 0.9666666388511658, "rewards/correctness/std": 0.1810203343629837, "rewards/length_penalty/mean": -0.20429687201976776, "rewards/length_penalty/std": 0.08035586029291153, "sampling/importance_sampling_ratio/max": 1.450929045677185, "sampling/importance_sampling_ratio/mean": 0.9924795031547546, "sampling/importance_sampling_ratio/min": 0.5934280157089233, "sampling/sampling_logp_difference/max": 0.5218393802642822, "sampling/sampling_logp_difference/mean": 0.01417568325996399, "step": 172, "step_time": 10.970934886718169 }, { "clip_ratio/high_max": 0.0006309296659310348, "clip_ratio/high_mean": 0.0006309296659310348, "clip_ratio/low_mean": 5.6718282091120877e-05, "clip_ratio/low_min": 5.6718282091120877e-05, "clip_ratio/region_mean": 0.0006876479480221557, "completions/clipped_ratio": 0.03333333507180214, "completions/max_length": 2048.0, "completions/max_terminated_length": 1610.0, "completions/mean_length": 570.4500122070312, "completions/mean_terminated_length": 519.5, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.2216138318181038, "epoch": 0.2521865889212828, "frac_reward_zero_std": 0.0, "grad_norm": 0.008872742764651775, "learning_rate": 2e-05, "loss": -0.0019448311068117619, "num_tokens": 9346215.0, "reward": 0.2047933042049408, "reward_std": 0.617644727230072, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.2785400450229645, "rewards/length_penalty/std": 0.19476625323295593, "sampling/importance_sampling_ratio/max": 1.6034185886383057, "sampling/importance_sampling_ratio/mean": 0.9922030568122864, "sampling/importance_sampling_ratio/min": 0.6622313261032104, "sampling/sampling_logp_difference/max": 0.4721379280090332, "sampling/sampling_logp_difference/mean": 0.01476544700562954, "step": 173, "step_time": 24.293195291189477 }, { "clip_ratio/high_max": 0.0005930817230061317, "clip_ratio/high_mean": 0.0005930817230061317, "clip_ratio/low_mean": 0.0001003987369282792, "clip_ratio/low_min": 0.0001003987369282792, "clip_ratio/region_mean": 0.0006934804647850493, "completions/clipped_ratio": 0.0, "completions/max_length": 1230.0, "completions/max_terminated_length": 1230.0, "completions/mean_length": 533.1666870117188, "completions/mean_terminated_length": 533.1666870117188, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.1814538538455963, "epoch": 0.2536443148688047, "frac_reward_zero_std": 0.0, "grad_norm": 0.012497815303504467, "learning_rate": 2e-05, "loss": 0.031932175159454346, "num_tokens": 9382015.0, "reward": 0.6729980707168579, "reward_std": 0.3128434121608734, "rewards/correctness/mean": 0.9333333373069763, "rewards/correctness/std": 0.2515488862991333, "rewards/length_penalty/mean": -0.2603352963924408, "rewards/length_penalty/std": 0.111988864839077, "sampling/importance_sampling_ratio/max": 1.600244402885437, "sampling/importance_sampling_ratio/mean": 0.993462085723877, "sampling/importance_sampling_ratio/min": 0.45217838883399963, "sampling/sampling_logp_difference/max": 0.7936785221099854, "sampling/sampling_logp_difference/mean": 0.012692712247371674, "step": 174, "step_time": 15.74416776606813 }, { "clip_ratio/high_max": 0.0005973838512242461, "clip_ratio/high_mean": 0.0005973838512242461, "clip_ratio/low_mean": 0.00025322781584691256, "clip_ratio/low_min": 0.00025322781584691256, "clip_ratio/region_mean": 0.0008506116670711587, "completions/clipped_ratio": 0.20000001788139343, "completions/max_length": 2048.0, "completions/max_terminated_length": 2044.0, "completions/mean_length": 1029.166748046875, "completions/mean_terminated_length": 774.4583740234375, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.30322639395793277, "epoch": 0.25510204081632654, "frac_reward_zero_std": 0.0, "grad_norm": 0.015394343994557858, "learning_rate": 2e-05, "loss": 0.0004860721528530121, "num_tokens": 9450505.0, "reward": 0.14747722446918488, "reward_std": 0.7596887350082397, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.5025227665901184, "rewards/length_penalty/std": 0.3278689980506897, "sampling/importance_sampling_ratio/max": 1.474064588546753, "sampling/importance_sampling_ratio/mean": 0.9891459345817566, "sampling/importance_sampling_ratio/min": 0.6035506725311279, "sampling/sampling_logp_difference/max": 0.5049252510070801, "sampling/sampling_logp_difference/mean": 0.019591135904192924, "step": 175, "step_time": 27.490358071168885 }, { "clip_ratio/high_max": 0.001002947586433341, "clip_ratio/high_mean": 0.001002947586433341, "clip_ratio/low_mean": 0.00013154309999663383, "clip_ratio/low_min": 0.00013154309999663383, "clip_ratio/region_mean": 0.0011344906912806134, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1703.0, "completions/mean_length": 1014.2500610351562, "completions/mean_terminated_length": 899.388916015625, "completions/min_length": 462.0, "completions/min_terminated_length": 462.0, "entropy": 0.40745416780312854, "epoch": 0.2565597667638484, "frac_reward_zero_std": 0.0, "grad_norm": 0.017374936491250992, "learning_rate": 2e-05, "loss": 0.04549677297472954, "num_tokens": 9517570.0, "reward": -0.09523925930261612, "reward_std": 0.6463657021522522, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.4952392578125, "rewards/length_penalty/std": 0.23023425042629242, "sampling/importance_sampling_ratio/max": 1.4509283304214478, "sampling/importance_sampling_ratio/mean": 0.9854934215545654, "sampling/importance_sampling_ratio/min": 0.47510966658592224, "sampling/sampling_logp_difference/max": 0.7442096471786499, "sampling/sampling_logp_difference/mean": 0.024702923372387886, "step": 176, "step_time": 27.569041184149683 }, { "clip_ratio/high_max": 0.0009653393450813988, "clip_ratio/high_mean": 0.0009653393450813988, "clip_ratio/low_mean": 8.393301201673846e-05, "clip_ratio/low_min": 8.393301201673846e-05, "clip_ratio/region_mean": 0.0010492723570981373, "completions/clipped_ratio": 0.0, "completions/max_length": 1833.0, "completions/max_terminated_length": 1833.0, "completions/mean_length": 585.8333740234375, "completions/mean_terminated_length": 585.8333740234375, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.314935398598512, "epoch": 0.25801749271137026, "frac_reward_zero_std": 0.0, "grad_norm": 0.01463930681347847, "learning_rate": 2e-05, "loss": -0.006962409242987633, "num_tokens": 9557790.0, "reward": 0.3139485716819763, "reward_std": 0.5434808731079102, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.2860514223575592, "rewards/length_penalty/std": 0.15339350700378418, "sampling/importance_sampling_ratio/max": 1.5201925039291382, "sampling/importance_sampling_ratio/mean": 0.9881151914596558, "sampling/importance_sampling_ratio/min": 0.6471298336982727, "sampling/sampling_logp_difference/max": 0.4352083206176758, "sampling/sampling_logp_difference/mean": 0.021870296448469162, "step": 177, "step_time": 22.817483997903764 }, { "clip_ratio/high_max": 0.001856392181556051, "clip_ratio/high_mean": 0.001856392181556051, "clip_ratio/low_mean": 0.00014501971600111574, "clip_ratio/low_min": 0.00014501971600111574, "clip_ratio/region_mean": 0.002001411863602698, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1962.0, "completions/mean_length": 705.0167236328125, "completions/mean_terminated_length": 682.2542114257812, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.2905154253045718, "epoch": 0.2594752186588921, "frac_reward_zero_std": 0.0, "grad_norm": 0.015073919668793678, "learning_rate": 2e-05, "loss": 0.023865118622779846, "num_tokens": 9604961.0, "reward": 0.3724202811717987, "reward_std": 0.5507521629333496, "rewards/correctness/mean": 0.7166666388511658, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.34424641728401184, "rewards/length_penalty/std": 0.1690424531698227, "sampling/importance_sampling_ratio/max": 1.9097141027450562, "sampling/importance_sampling_ratio/mean": 0.9898587465286255, "sampling/importance_sampling_ratio/min": 0.6631399393081665, "sampling/sampling_logp_difference/max": 0.6469535827636719, "sampling/sampling_logp_difference/mean": 0.018903009593486786, "step": 178, "step_time": 25.403236869955435 }, { "clip_ratio/high_max": 0.0015735746322510142, "clip_ratio/high_mean": 0.0015735746322510142, "clip_ratio/low_mean": 0.00014540493915167949, "clip_ratio/low_min": 0.00014540493915167949, "clip_ratio/region_mean": 0.0017189795811039705, "completions/clipped_ratio": 0.0, "completions/max_length": 2020.0, "completions/max_terminated_length": 2020.0, "completions/mean_length": 600.7833862304688, "completions/mean_terminated_length": 600.7833862304688, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 0.26812274505694705, "epoch": 0.260932944606414, "frac_reward_zero_std": 0.0, "grad_norm": 0.013311340473592281, "learning_rate": 2e-05, "loss": 0.021306850016117096, "num_tokens": 9645838.0, "reward": 0.30664879083633423, "reward_std": 0.5342420339584351, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4940321743488312, "rewards/length_penalty/mean": -0.2933512330055237, "rewards/length_penalty/std": 0.188986137509346, "sampling/importance_sampling_ratio/max": 1.5045559406280518, "sampling/importance_sampling_ratio/mean": 0.9907915592193604, "sampling/importance_sampling_ratio/min": 0.6281037330627441, "sampling/sampling_logp_difference/max": 0.46504998207092285, "sampling/sampling_logp_difference/mean": 0.01711450330913067, "step": 179, "step_time": 24.340147527866066 }, { "clip_ratio/high_max": 0.000845977726082007, "clip_ratio/high_mean": 0.000845977726082007, "clip_ratio/low_mean": 0.0001156907164840959, "clip_ratio/low_min": 0.0001156907164840959, "clip_ratio/region_mean": 0.0009616684498420606, "completions/clipped_ratio": 0.0, "completions/max_length": 1717.0, "completions/max_terminated_length": 1717.0, "completions/mean_length": 743.8500366210938, "completions/mean_terminated_length": 743.8500366210938, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.28752462814251584, "epoch": 0.26239067055393583, "frac_reward_zero_std": 0.0, "grad_norm": 0.012962594628334045, "learning_rate": 2e-05, "loss": 0.0020775552839040756, "num_tokens": 9694649.0, "reward": 0.1701253354549408, "reward_std": 0.5588796734809875, "rewards/correctness/mean": 0.5333333611488342, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.36320799589157104, "rewards/length_penalty/std": 0.15787248313426971, "sampling/importance_sampling_ratio/max": 1.4653091430664062, "sampling/importance_sampling_ratio/mean": 0.9898093342781067, "sampling/importance_sampling_ratio/min": 0.6383785009384155, "sampling/sampling_logp_difference/max": 0.4488239288330078, "sampling/sampling_logp_difference/mean": 0.018891487270593643, "step": 180, "step_time": 22.350864383857697 }, { "clip_ratio/high_max": 0.001120295113651082, "clip_ratio/high_mean": 0.001120295113651082, "clip_ratio/low_mean": 6.471350692057361e-05, "clip_ratio/low_min": 6.471350692057361e-05, "clip_ratio/region_mean": 0.0011850086060197402, "completions/clipped_ratio": 0.05000000447034836, "completions/max_length": 2048.0, "completions/max_terminated_length": 1806.0, "completions/mean_length": 878.3833618164062, "completions/mean_terminated_length": 816.8245849609375, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.2872309908270836, "epoch": 0.26384839650145775, "frac_reward_zero_std": 0.0, "grad_norm": 0.0205439031124115, "learning_rate": 2e-05, "loss": 0.06915633380413055, "num_tokens": 9751932.0, "reward": 0.4211018979549408, "reward_std": 0.5352883338928223, "rewards/correctness/mean": 0.8500000238418579, "rewards/correctness/std": 0.36008474230766296, "rewards/length_penalty/mean": -0.4288981258869171, "rewards/length_penalty/std": 0.23621498048305511, "sampling/importance_sampling_ratio/max": 1.4063832759857178, "sampling/importance_sampling_ratio/mean": 0.9895955324172974, "sampling/importance_sampling_ratio/min": 0.6374525427818298, "sampling/sampling_logp_difference/max": 0.4502754211425781, "sampling/sampling_logp_difference/mean": 0.018434900790452957, "step": 181, "step_time": 25.942147817928344 }, { "clip_ratio/high_max": 0.0006055711128283292, "clip_ratio/high_mean": 0.0006055711128283292, "clip_ratio/low_mean": 0.00015449593047378585, "clip_ratio/low_min": 0.00015449593047378585, "clip_ratio/region_mean": 0.0007600670360261574, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 834.0, "completions/mean_length": 663.0167236328125, "completions/mean_terminated_length": 537.1090698242188, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.2742292533318202, "epoch": 0.2653061224489796, "frac_reward_zero_std": 0.0, "grad_norm": 0.015478339046239853, "learning_rate": 2e-05, "loss": 0.0617741234600544, "num_tokens": 9796143.0, "reward": 0.32626140117645264, "reward_std": 0.6145201921463013, "rewards/correctness/mean": 0.6499999761581421, "rewards/correctness/std": 0.48099473118782043, "rewards/length_penalty/mean": -0.32373860478401184, "rewards/length_penalty/std": 0.21554671227931976, "sampling/importance_sampling_ratio/max": 1.8416732549667358, "sampling/importance_sampling_ratio/mean": 0.9902825355529785, "sampling/importance_sampling_ratio/min": 0.4849720895290375, "sampling/sampling_logp_difference/max": 0.7236639261245728, "sampling/sampling_logp_difference/mean": 0.01882294937968254, "step": 182, "step_time": 24.860169053077698 }, { "clip_ratio/high_max": 0.0012968556402483955, "clip_ratio/high_mean": 0.0012968556402483955, "clip_ratio/low_mean": 0.00016336666885763407, "clip_ratio/low_min": 0.00016336666885763407, "clip_ratio/region_mean": 0.0014602223091060296, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1839.0, "completions/mean_length": 564.8500366210938, "completions/mean_terminated_length": 539.7118530273438, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.3727461298306783, "epoch": 0.26676384839650147, "frac_reward_zero_std": 0.0, "grad_norm": 0.01920958235859871, "learning_rate": 2e-05, "loss": 0.033165499567985535, "num_tokens": 9833634.0, "reward": 0.14086100459098816, "reward_std": 0.5286503434181213, "rewards/correctness/mean": 0.4166666567325592, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.27580565214157104, "rewards/length_penalty/std": 0.18563447892665863, "sampling/importance_sampling_ratio/max": 1.4753550291061401, "sampling/importance_sampling_ratio/mean": 0.9871658682823181, "sampling/importance_sampling_ratio/min": 0.6304611563682556, "sampling/sampling_logp_difference/max": 0.4613037109375, "sampling/sampling_logp_difference/mean": 0.022531885653734207, "step": 183, "step_time": 24.735460065072402 }, { "clip_ratio/high_max": 0.0008379968882460768, "clip_ratio/high_mean": 0.0008379968882460768, "clip_ratio/low_mean": 6.425392105787371e-05, "clip_ratio/low_min": 6.425392105787371e-05, "clip_ratio/region_mean": 0.0009022508165799081, "completions/clipped_ratio": 0.0, "completions/max_length": 1879.0, "completions/max_terminated_length": 1879.0, "completions/mean_length": 744.4667358398438, "completions/mean_terminated_length": 744.4667358398438, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "entropy": 0.1703206648429235, "epoch": 0.26822157434402333, "frac_reward_zero_std": 0.0, "grad_norm": 0.01721379905939102, "learning_rate": 2e-05, "loss": -0.0008378904312849045, "num_tokens": 9883722.0, "reward": 0.36982423067092896, "reward_std": 0.4271937906742096, "rewards/correctness/mean": 0.7333333492279053, "rewards/correctness/std": 0.4459484815597534, "rewards/length_penalty/mean": -0.3635091185569763, "rewards/length_penalty/std": 0.2257184088230133, "sampling/importance_sampling_ratio/max": 1.4273576736450195, "sampling/importance_sampling_ratio/mean": 0.9935067892074585, "sampling/importance_sampling_ratio/min": 0.4246005117893219, "sampling/sampling_logp_difference/max": 0.8566066026687622, "sampling/sampling_logp_difference/mean": 0.012150940485298634, "step": 184, "step_time": 23.781651353929192 }, { "clip_ratio/high_max": 0.000999829791301939, "clip_ratio/high_mean": 0.000999829791301939, "clip_ratio/low_mean": 0.0002239340950230447, "clip_ratio/low_min": 0.0002239340950230447, "clip_ratio/region_mean": 0.0012237638802616857, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1923.0, "completions/mean_length": 921.5833740234375, "completions/mean_terminated_length": 748.2885131835938, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.4105442514022191, "epoch": 0.2696793002915452, "frac_reward_zero_std": 0.0, "grad_norm": 0.01904107816517353, "learning_rate": 2e-05, "loss": 0.10748829692602158, "num_tokens": 9944517.0, "reward": 0.11667481064796448, "reward_std": 0.6949194073677063, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971747398376465, "rewards/length_penalty/mean": -0.4499918520450592, "rewards/length_penalty/std": 0.2978811264038086, "sampling/importance_sampling_ratio/max": 1.4936118125915527, "sampling/importance_sampling_ratio/mean": 0.9865139722824097, "sampling/importance_sampling_ratio/min": 0.47563496232032776, "sampling/sampling_logp_difference/max": 0.7431045770645142, "sampling/sampling_logp_difference/mean": 0.024014528840780258, "step": 185, "step_time": 26.656971606891602 }, { "clip_ratio/high_max": 0.00039769137583789416, "clip_ratio/high_mean": 0.00039769137583789416, "clip_ratio/low_mean": 0.0002477293007056384, "clip_ratio/low_min": 0.0002477293007056384, "clip_ratio/region_mean": 0.0006454206741182134, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1886.0, "completions/mean_length": 832.6000366210938, "completions/mean_terminated_length": 645.6154174804688, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.21153807391722998, "epoch": 0.27113702623906705, "frac_reward_zero_std": 0.0, "grad_norm": 0.01534296479076147, "learning_rate": 2e-05, "loss": 0.034561239182949066, "num_tokens": 9998603.0, "reward": 0.11012370139360428, "reward_std": 0.7228626608848572, "rewards/correctness/mean": 0.5166666507720947, "rewards/correctness/std": 0.5039393305778503, "rewards/length_penalty/mean": -0.40654295682907104, "rewards/length_penalty/std": 0.29053258895874023, "sampling/importance_sampling_ratio/max": 1.6479967832565308, "sampling/importance_sampling_ratio/mean": 0.991916835308075, "sampling/importance_sampling_ratio/min": 0.520044207572937, "sampling/sampling_logp_difference/max": 0.653841495513916, "sampling/sampling_logp_difference/mean": 0.014898436143994331, "step": 186, "step_time": 27.10042130202055 }, { "clip_ratio/high_max": 0.000987601451925002, "clip_ratio/high_mean": 0.000987601451925002, "clip_ratio/low_mean": 0.0003099991808994673, "clip_ratio/low_min": 0.0003099991808994673, "clip_ratio/region_mean": 0.0012976006158472349, "completions/clipped_ratio": 0.38333335518836975, "completions/max_length": 2048.0, "completions/max_terminated_length": 1950.0, "completions/mean_length": 1271.166748046875, "completions/mean_terminated_length": 788.270263671875, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.5849319398403168, "epoch": 0.2725947521865889, "frac_reward_zero_std": 0.0, "grad_norm": 0.02560403384268284, "learning_rate": 2e-05, "loss": 0.044088806957006454, "num_tokens": 10081043.0, "reward": -0.10402018576860428, "reward_std": 0.8494263291358948, "rewards/correctness/mean": 0.5166666507720947, "rewards/correctness/std": 0.5039393305778503, "rewards/length_penalty/mean": -0.6206868290901184, "rewards/length_penalty/std": 0.3692076802253723, "sampling/importance_sampling_ratio/max": 1.7236531972885132, "sampling/importance_sampling_ratio/mean": 0.980085551738739, "sampling/importance_sampling_ratio/min": 0.6232811212539673, "sampling/sampling_logp_difference/max": 0.5444459915161133, "sampling/sampling_logp_difference/mean": 0.034810230135917664, "step": 187, "step_time": 28.50091272802092 }, { "clip_ratio/high_max": 0.0009432584435368577, "clip_ratio/high_mean": 0.0009432584435368577, "clip_ratio/low_mean": 0.0001643827126827091, "clip_ratio/low_min": 0.0001643827126827091, "clip_ratio/region_mean": 0.00110764114651829, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1876.0, "completions/mean_length": 950.4334106445312, "completions/mean_terminated_length": 781.5769653320312, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.37916910151640576, "epoch": 0.27405247813411077, "frac_reward_zero_std": 0.0, "grad_norm": 0.017173759639263153, "learning_rate": 2e-05, "loss": 0.016559267416596413, "num_tokens": 10142859.0, "reward": 0.03592122718691826, "reward_std": 0.6352493166923523, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5042194724082947, "rewards/length_penalty/mean": -0.46407878398895264, "rewards/length_penalty/std": 0.2751929461956024, "sampling/importance_sampling_ratio/max": 1.5932568311691284, "sampling/importance_sampling_ratio/mean": 0.98691725730896, "sampling/importance_sampling_ratio/min": 0.5883161425590515, "sampling/sampling_logp_difference/max": 0.5304908752441406, "sampling/sampling_logp_difference/mean": 0.023954246193170547, "step": 188, "step_time": 27.15778080979362 }, { "clip_ratio/high_max": 0.0006955851373883585, "clip_ratio/high_mean": 0.0006955851373883585, "clip_ratio/low_mean": 0.00011439093577791937, "clip_ratio/low_min": 0.00011439093577791937, "clip_ratio/region_mean": 0.0008099760743789375, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1836.0, "completions/mean_length": 923.300048828125, "completions/mean_terminated_length": 798.3333129882812, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.2857773353656133, "epoch": 0.2755102040816326, "frac_reward_zero_std": 0.0, "grad_norm": 0.01848120614886284, "learning_rate": 2e-05, "loss": 0.030527060851454735, "num_tokens": 10202417.0, "reward": 0.2325032651424408, "reward_std": 0.6638219952583313, "rewards/correctness/mean": 0.6833333373069763, "rewards/correctness/std": 0.46910181641578674, "rewards/length_penalty/mean": -0.4508300721645355, "rewards/length_penalty/std": 0.2595711648464203, "sampling/importance_sampling_ratio/max": 1.5372258424758911, "sampling/importance_sampling_ratio/mean": 0.9894238710403442, "sampling/importance_sampling_ratio/min": 0.6098623275756836, "sampling/sampling_logp_difference/max": 0.4945220947265625, "sampling/sampling_logp_difference/mean": 0.01942998357117176, "step": 189, "step_time": 26.307919850572944 }, { "clip_ratio/high_max": 0.001184566431523611, "clip_ratio/high_mean": 0.001184566431523611, "clip_ratio/low_mean": 0.00018724674737313762, "clip_ratio/low_min": 0.00018724674737313762, "clip_ratio/region_mean": 0.001371813181322068, "completions/clipped_ratio": 0.18333333730697632, "completions/max_length": 2048.0, "completions/max_terminated_length": 1692.0, "completions/mean_length": 1109.25, "completions/mean_terminated_length": 898.5101928710938, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.5130080531040827, "epoch": 0.27696793002915454, "frac_reward_zero_std": 0.0, "grad_norm": 0.018907280638813972, "learning_rate": 2e-05, "loss": 0.03437569737434387, "num_tokens": 10274082.0, "reward": -0.05829264596104622, "reward_std": 0.6425654292106628, "rewards/correctness/mean": 0.4833333194255829, "rewards/correctness/std": 0.5039392709732056, "rewards/length_penalty/mean": -0.5416259765625, "rewards/length_penalty/std": 0.2773684859275818, "sampling/importance_sampling_ratio/max": 1.6074111461639404, "sampling/importance_sampling_ratio/mean": 0.9823446869850159, "sampling/importance_sampling_ratio/min": 0.6308839917182922, "sampling/sampling_logp_difference/max": 0.4746248722076416, "sampling/sampling_logp_difference/mean": 0.031214483082294464, "step": 190, "step_time": 27.215038079069927 }, { "clip_ratio/high_max": 0.0009124269078408057, "clip_ratio/high_mean": 0.0009124269078408057, "clip_ratio/low_mean": 0.00021373673977601962, "clip_ratio/low_min": 0.00021373673977601962, "clip_ratio/region_mean": 0.0011261636197256546, "completions/clipped_ratio": 0.15000000596046448, "completions/max_length": 2048.0, "completions/max_terminated_length": 1815.0, "completions/mean_length": 842.4000244140625, "completions/mean_terminated_length": 629.6470947265625, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.42648114264011383, "epoch": 0.2784256559766764, "frac_reward_zero_std": 0.0, "grad_norm": 0.017047202214598656, "learning_rate": 2e-05, "loss": 0.005941006354987621, "num_tokens": 10329286.0, "reward": 0.17200522124767303, "reward_std": 0.6729675531387329, "rewards/correctness/mean": 0.5833333134651184, "rewards/correctness/std": 0.4971671402454376, "rewards/length_penalty/mean": -0.41132813692092896, "rewards/length_penalty/std": 0.28831717371940613, "sampling/importance_sampling_ratio/max": 1.7022995948791504, "sampling/importance_sampling_ratio/mean": 0.9851097464561462, "sampling/importance_sampling_ratio/min": 0.6182209253311157, "sampling/sampling_logp_difference/max": 0.531980037689209, "sampling/sampling_logp_difference/mean": 0.027465863153338432, "step": 191, "step_time": 26.72552009532228 }, { "clip_ratio/high_max": 0.001399731185908119, "clip_ratio/high_mean": 0.001399731185908119, "clip_ratio/low_mean": 7.817385873446862e-05, "clip_ratio/low_min": 7.817385873446862e-05, "clip_ratio/region_mean": 0.0014779050446425874, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1720.0, "completions/mean_length": 572.0667114257812, "completions/mean_terminated_length": 547.0508422851562, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.33766258011261624, "epoch": 0.27988338192419826, "frac_reward_zero_std": 0.0, "grad_norm": 0.023948965594172478, "learning_rate": 2e-05, "loss": 0.06839627027511597, "num_tokens": 10369010.0, "reward": 0.2873372435569763, "reward_std": 0.5584322214126587, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.27932941913604736, "rewards/length_penalty/std": 0.18996958434581757, "sampling/importance_sampling_ratio/max": 1.6533702611923218, "sampling/importance_sampling_ratio/mean": 0.9873671531677246, "sampling/importance_sampling_ratio/min": 0.5788065195083618, "sampling/sampling_logp_difference/max": 0.5467870235443115, "sampling/sampling_logp_difference/mean": 0.02334241196513176, "step": 192, "step_time": 24.896456439048052 }, { "clip_ratio/high_max": 0.0007610242513086026, "clip_ratio/high_mean": 0.0007610242513086026, "clip_ratio/low_mean": 0.0002833080046305743, "clip_ratio/low_min": 0.0002833080046305743, "clip_ratio/region_mean": 0.0010443322389619425, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 2048.0, "completions/mean_length": 871.2000732421875, "completions/mean_terminated_length": 740.4444580078125, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.3991595556338628, "epoch": 0.2813411078717201, "frac_reward_zero_std": 0.0, "grad_norm": 0.022390838712453842, "learning_rate": 2e-05, "loss": 0.06970308721065521, "num_tokens": 10427492.0, "reward": 0.04127604514360428, "reward_std": 0.6397515535354614, "rewards/correctness/mean": 0.46666666865348816, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.4253906309604645, "rewards/length_penalty/std": 0.3054787814617157, "sampling/importance_sampling_ratio/max": 1.798714518547058, "sampling/importance_sampling_ratio/mean": 0.9861185550689697, "sampling/importance_sampling_ratio/min": 0.5218219757080078, "sampling/sampling_logp_difference/max": 0.6504287719726562, "sampling/sampling_logp_difference/mean": 0.025752553716301918, "step": 193, "step_time": 27.298154526157305 }, { "clip_ratio/high_max": 0.0005429390585049987, "clip_ratio/high_mean": 0.0005429390585049987, "clip_ratio/low_mean": 0.00016068309923866764, "clip_ratio/low_min": 0.00016068309923866764, "clip_ratio/region_mean": 0.0007036221407664319, "completions/clipped_ratio": 0.01666666753590107, "completions/max_length": 2048.0, "completions/max_terminated_length": 1571.0, "completions/mean_length": 465.9833679199219, "completions/mean_terminated_length": 439.16949462890625, "completions/min_length": 112.0, "completions/min_terminated_length": 112.0, "entropy": 0.25524456550677616, "epoch": 0.282798833819242, "frac_reward_zero_std": 0.0, "grad_norm": 0.018759645521640778, "learning_rate": 2e-05, "loss": 0.010321836918592453, "num_tokens": 10459081.0, "reward": 0.6891357898712158, "reward_std": 0.3848750591278076, "rewards/correctness/mean": 0.9166666865348816, "rewards/correctness/std": 0.2787178158760071, "rewards/length_penalty/mean": -0.22753092646598816, "rewards/length_penalty/std": 0.15539667010307312, "sampling/importance_sampling_ratio/max": 1.6560379266738892, "sampling/importance_sampling_ratio/mean": 0.990645170211792, "sampling/importance_sampling_ratio/min": 0.6468564867973328, "sampling/sampling_logp_difference/max": 0.5044279098510742, "sampling/sampling_logp_difference/mean": 0.018840471282601357, "step": 194, "step_time": 23.62958171311766 }, { "clip_ratio/high_max": 0.0015640522178728133, "clip_ratio/high_mean": 0.0015640522178728133, "clip_ratio/low_mean": 0.00019902208684167513, "clip_ratio/low_min": 0.00019902208684167513, "clip_ratio/region_mean": 0.0017630742610587429, "completions/clipped_ratio": 0.13333334028720856, "completions/max_length": 2048.0, "completions/max_terminated_length": 1778.0, "completions/mean_length": 976.2333984375, "completions/mean_terminated_length": 811.34619140625, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.44658708075682324, "epoch": 0.28425655976676384, "frac_reward_zero_std": 0.0, "grad_norm": 0.021446384489536285, "learning_rate": 2e-05, "loss": 0.05115346610546112, "num_tokens": 10522335.0, "reward": 0.22332358360290527, "reward_std": 0.670845091342926, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4621247947216034, "rewards/length_penalty/mean": -0.47667643427848816, "rewards/length_penalty/std": 0.2933204472064972, "sampling/importance_sampling_ratio/max": 1.6302381753921509, "sampling/importance_sampling_ratio/mean": 0.9849187731742859, "sampling/importance_sampling_ratio/min": 0.5913414359092712, "sampling/sampling_logp_difference/max": 0.5253617763519287, "sampling/sampling_logp_difference/mean": 0.027165725827217102, "step": 195, "step_time": 27.863610920263454 }, { "clip_ratio/high_max": 0.0016187613170283537, "clip_ratio/high_mean": 0.0016187613170283537, "clip_ratio/low_mean": 0.00023703551172123602, "clip_ratio/low_min": 0.00023703551172123602, "clip_ratio/region_mean": 0.0018557968433015049, "completions/clipped_ratio": 0.0833333358168602, "completions/max_length": 2048.0, "completions/max_terminated_length": 2008.0, "completions/mean_length": 949.6500244140625, "completions/mean_terminated_length": 849.7999877929688, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.4802217831214269, "epoch": 0.2857142857142857, "frac_reward_zero_std": 0.0, "grad_norm": 0.023930514231324196, "learning_rate": 2e-05, "loss": 0.08557190001010895, "num_tokens": 10583064.0, "reward": 0.30297037959098816, "reward_std": 0.6304841637611389, "rewards/correctness/mean": 0.7666666507720947, "rewards/correctness/std": 0.42652183771133423, "rewards/length_penalty/mean": -0.46369630098342896, "rewards/length_penalty/std": 0.27533650398254395, "sampling/importance_sampling_ratio/max": 1.991895079612732, "sampling/importance_sampling_ratio/mean": 0.9830731153488159, "sampling/importance_sampling_ratio/min": 0.568490207195282, "sampling/sampling_logp_difference/max": 0.6890864372253418, "sampling/sampling_logp_difference/mean": 0.03178530931472778, "step": 196, "step_time": 26.212359979748726 }, { "clip_ratio/high_max": 0.000976633057386304, "clip_ratio/high_mean": 0.000976633057386304, "clip_ratio/low_mean": 0.0002062602531320105, "clip_ratio/low_min": 0.0002062602531320105, "clip_ratio/region_mean": 0.0011828933202195913, "completions/clipped_ratio": 0.2666666805744171, "completions/max_length": 2048.0, "completions/max_terminated_length": 2022.0, "completions/mean_length": 1065.0167236328125, "completions/mean_terminated_length": 707.5681762695312, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 0.45353777209917706, "epoch": 0.28717201166180756, "frac_reward_zero_std": 0.0, "grad_norm": 0.01922941952943802, "learning_rate": 2e-05, "loss": 0.06035600230097771, "num_tokens": 10654885.0, "reward": 0.04663899913430214, "reward_std": 0.7512121200561523, "rewards/correctness/mean": 0.5666666626930237, "rewards/correctness/std": 0.49971744418144226, "rewards/length_penalty/mean": -0.5200276970863342, "rewards/length_penalty/std": 0.3408592641353607, "sampling/importance_sampling_ratio/max": 1.76103937625885, "sampling/importance_sampling_ratio/mean": 0.9833039045333862, "sampling/importance_sampling_ratio/min": 0.5540443062782288, "sampling/sampling_logp_difference/max": 0.5905106067657471, "sampling/sampling_logp_difference/mean": 0.03049941174685955, "step": 197, "step_time": 29.642499959561974 }, { "clip_ratio/high_max": 0.0006672914654094105, "clip_ratio/high_mean": 0.0006672914654094105, "clip_ratio/low_mean": 0.00011224134747559826, "clip_ratio/low_min": 0.00011224134747559826, "clip_ratio/region_mean": 0.0007795328128850088, "completions/clipped_ratio": 0.0, "completions/max_length": 1777.0, "completions/max_terminated_length": 1777.0, "completions/mean_length": 620.1000366210938, "completions/mean_terminated_length": 620.1000366210938, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.22741481413443884, "epoch": 0.2886297376093295, "frac_reward_zero_std": 0.0, "grad_norm": 0.01049730647355318, "learning_rate": 2e-05, "loss": -0.0004327157512307167, "num_tokens": 10696041.0, "reward": 0.4138834774494171, "reward_std": 0.42144575715065, "rewards/correctness/mean": 0.7166666388511658, "rewards/correctness/std": 0.45441964268684387, "rewards/length_penalty/mean": -0.30278319120407104, "rewards/length_penalty/std": 0.2068432718515396, "sampling/importance_sampling_ratio/max": 1.6680134534835815, "sampling/importance_sampling_ratio/mean": 0.9916688799858093, "sampling/importance_sampling_ratio/min": 0.6686152815818787, "sampling/sampling_logp_difference/max": 0.5116333961486816, "sampling/sampling_logp_difference/mean": 0.016084833070635796, "step": 198, "step_time": 21.738332585897297 }, { "clip_ratio/high_max": 0.0010048883559647948, "clip_ratio/high_mean": 0.0010048883559647948, "clip_ratio/low_mean": 0.00013046783957785615, "clip_ratio/low_min": 0.00013046783957785615, "clip_ratio/region_mean": 0.0011353562003932893, "completions/clipped_ratio": 0.0, "completions/max_length": 1556.0, "completions/max_terminated_length": 1556.0, "completions/mean_length": 610.3500366210938, "completions/mean_terminated_length": 610.3500366210938, "completions/min_length": 99.0, "completions/min_terminated_length": 99.0, "entropy": 0.23824912309646606, "epoch": 0.29008746355685133, "frac_reward_zero_std": 0.0, "grad_norm": 0.011849445290863514, "learning_rate": 2e-05, "loss": 0.004184707999229431, "num_tokens": 10737212.0, "reward": 0.36864423751831055, "reward_std": 0.5543534755706787, "rewards/correctness/mean": 0.6666666865348816, "rewards/correctness/std": 0.4753827154636383, "rewards/length_penalty/mean": -0.29802244901657104, "rewards/length_penalty/std": 0.16635596752166748, "sampling/importance_sampling_ratio/max": 1.4651864767074585, "sampling/importance_sampling_ratio/mean": 0.9918603301048279, "sampling/importance_sampling_ratio/min": 0.6047338843345642, "sampling/sampling_logp_difference/max": 0.5029667615890503, "sampling/sampling_logp_difference/mean": 0.016273291781544685, "step": 199, "step_time": 19.38298167823814 }, { "clip_ratio/high_max": 0.001253287142996366, "clip_ratio/high_mean": 0.001253287142996366, "clip_ratio/low_mean": 0.00022714178824874884, "clip_ratio/low_min": 0.00022714178824874884, "clip_ratio/region_mean": 0.0014804289482223492, "completions/clipped_ratio": 0.10000000894069672, "completions/max_length": 2048.0, "completions/max_terminated_length": 1888.0, "completions/mean_length": 940.7167358398438, "completions/mean_terminated_length": 817.6851806640625, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.35443704823652905, "epoch": 0.2915451895043732, "frac_reward_zero_std": 0.0, "grad_norm": 0.02682001329958439, "learning_rate": 2e-05, "loss": 0.06590258330106735, "num_tokens": 10798505.0, "reward": 0.07399902492761612, "reward_std": 0.6956524848937988, "rewards/correctness/mean": 0.5333333611488342, "rewards/correctness/std": 0.5030977725982666, "rewards/length_penalty/mean": -0.4593343138694763, "rewards/length_penalty/std": 0.25864270329475403, "sampling/importance_sampling_ratio/max": 1.6371389627456665, "sampling/importance_sampling_ratio/mean": 0.9873014092445374, "sampling/importance_sampling_ratio/min": 0.6002986431121826, "sampling/sampling_logp_difference/max": 0.5103280544281006, "sampling/sampling_logp_difference/mean": 0.02298104204237461, "step": 200, "step_time": 26.898668461013585 } ], "logging_steps": 1, "max_steps": 1200, "num_input_tokens_seen": 10798505, "num_train_epochs": 2, "save_steps": 200, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 10, "trial_name": null, "trial_params": null }