{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 2.0, "eval_steps": 500, "global_step": 1350, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.00024123263792716899, "clip_ratio/high_mean": 0.00024123263792716899, "clip_ratio/low_mean": 9.1552734375e-05, "clip_ratio/low_min": 9.1552734375e-05, "clip_ratio/region_mean": 0.00033278537230216896, "completions/clipped_ratio": 0.2875, "completions/max_length": 1805.2, "completions/max_terminated_length": 1306.0, "completions/mean_length": 1381.7125, "completions/mean_terminated_length": 1010.5482299804687, "completions/min_length": 1012.2, "completions/min_terminated_length": 807.4, "entropy": 0.3070180159062147, "epoch": 0.014814814814814815, "frac_reward_zero_std": 0.85, "grad_norm": 0.0, "learning_rate": 2.2167487684729066e-07, "loss": -0.01720508635044098, "num_tokens": 122133.0, "reward": 0.6762500122189522, "reward_std": 0.3998018682003021, "rewards/correctness_reward/mean": 0.6050000086426734, "rewards/correctness_reward/std": 0.37315957248210907, "rewards/format_reward/mean": 0.07125000245869159, "rewards/format_reward/std": 0.02664227820932865, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 0.8657017722725868, "sampling/importance_sampling_ratio/mean": 0.2813446238636971, "sampling/importance_sampling_ratio/min": 0.010075021703960374, "sampling/sampling_logp_difference/max": 0.7204151630401612, "sampling/sampling_logp_difference/mean": 0.01940987054258585, "step": 10, "step_time": 39.59839573744684 }, { "clip_ratio/high_max": 0.00047582906408933924, "clip_ratio/high_mean": 0.00047582906408933924, "clip_ratio/low_mean": 0.0020083684823475776, "clip_ratio/low_min": 0.0020083684823475776, "clip_ratio/region_mean": 0.002484197546436917, "completions/clipped_ratio": 0.6375, "completions/max_length": 2048.0, "completions/max_terminated_length": 1536.8, "completions/mean_length": 1810.2, "completions/mean_terminated_length": 1380.6816772460938, "completions/min_length": 1443.5, "completions/min_terminated_length": 1238.7, "entropy": 0.2902995118871331, "epoch": 0.02962962962962963, "frac_reward_zero_std": 0.6, "grad_norm": 0.0, "learning_rate": 4.6798029556650246e-07, "loss": 0.01638796627521515, "num_tokens": 279449.0, "reward": 0.20875000655651094, "reward_std": 0.5146574378013611, "rewards/correctness_reward/mean": 0.17250000350177289, "rewards/correctness_reward/std": 0.4749233454465866, "rewards/format_reward/mean": 0.03625000044703484, "rewards/format_reward/std": 0.04326232112944126, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 0.9496106281876564, "sampling/importance_sampling_ratio/mean": 0.22828449606895446, "sampling/importance_sampling_ratio/min": 0.0022496526275062934, "sampling/sampling_logp_difference/max": 0.8016536951065063, "sampling/sampling_logp_difference/mean": 0.018761485628783704, "step": 20, "step_time": 45.29714843314141 }, { "clip_ratio/high_max": 0.0007201705695479177, "clip_ratio/high_mean": 0.0007201705695479177, "clip_ratio/low_mean": 0.0014892578125, "clip_ratio/low_min": 0.0014892578125, "clip_ratio/region_mean": 0.002209428382047918, "completions/clipped_ratio": 0.575, "completions/max_length": 2017.4, "completions/max_terminated_length": 1351.1, "completions/mean_length": 1767.425, "completions/mean_terminated_length": 1151.6553588867187, "completions/min_length": 1369.2, "completions/min_terminated_length": 959.6, "entropy": 0.299412889778614, "epoch": 0.044444444444444446, "frac_reward_zero_std": 0.6, "grad_norm": 0.0, "learning_rate": 7.142857142857143e-07, "loss": 0.017733766138553618, "num_tokens": 433131.0, "reward": 0.2312500063329935, "reward_std": 0.37404934614896773, "rewards/correctness_reward/mean": 0.18749999888241292, "rewards/correctness_reward/std": 0.3411480985581875, "rewards/format_reward/mean": 0.04375000111758709, "rewards/format_reward/std": 0.036616604030132296, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 0.9527138769626617, "sampling/importance_sampling_ratio/mean": 0.21556818038225173, "sampling/importance_sampling_ratio/min": 0.0010286853241268546, "sampling/sampling_logp_difference/max": 0.9188494324684143, "sampling/sampling_logp_difference/mean": 0.019727922230958938, "step": 30, "step_time": 44.624646625109015 }, { "clip_ratio/high_max": 0.0003145492111798376, "clip_ratio/high_mean": 0.0003145492111798376, "clip_ratio/low_mean": 0.0006974496587645263, "clip_ratio/low_min": 0.0006974496587645263, "clip_ratio/region_mean": 0.0010119988699443638, "completions/clipped_ratio": 0.55, "completions/max_length": 1891.3, "completions/max_terminated_length": 1326.1, "completions/mean_length": 1728.975, "completions/mean_terminated_length": 1182.3875122070312, "completions/min_length": 1438.5, "completions/min_terminated_length": 1028.9, "entropy": 0.29536315277218816, "epoch": 0.05925925925925926, "frac_reward_zero_std": 0.8, "grad_norm": 0.0, "learning_rate": 9.605911330049262e-07, "loss": 0.004931902512907982, "num_tokens": 583861.0, "reward": 0.2975000113248825, "reward_std": 0.2702170670032501, "rewards/correctness_reward/mean": 0.2524999976158142, "rewards/correctness_reward/std": 0.24828823655843735, "rewards/format_reward/mean": 0.04500000085681677, "rewards/format_reward/std": 0.022767277806997298, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3339441269636154, "sampling/importance_sampling_ratio/mean": 0.30694490000605584, "sampling/importance_sampling_ratio/min": 0.008654238318558782, "sampling/sampling_logp_difference/max": 0.7304492771625519, "sampling/sampling_logp_difference/mean": 0.018808723613619804, "step": 40, "step_time": 42.38297662269324 }, { "clip_ratio/high_max": 0.000330584729090333, "clip_ratio/high_mean": 0.000330584729090333, "clip_ratio/low_mean": 0.00133056640625, "clip_ratio/low_min": 0.00133056640625, "clip_ratio/region_mean": 0.001661151135340333, "completions/clipped_ratio": 0.75, "completions/max_length": 2048.0, "completions/max_terminated_length": 1210.9, "completions/mean_length": 1845.2375, "completions/mean_terminated_length": 1098.9833374023438, "completions/min_length": 1435.1, "completions/min_terminated_length": 1025.5, "entropy": 0.30036053210496905, "epoch": 0.07407407407407407, "frac_reward_zero_std": 0.7, "grad_norm": 0.0, "learning_rate": 1.2068965517241381e-06, "loss": -0.00017195791006088256, "num_tokens": 744136.0, "reward": 0.08750000670552253, "reward_std": 0.36153389811515807, "rewards/correctness_reward/mean": 0.062499997392296794, "rewards/correctness_reward/std": 0.327443091571331, "rewards/format_reward/mean": 0.025000000558793544, "rewards/format_reward/std": 0.03409078791737556, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.54177483022213, "sampling/importance_sampling_ratio/mean": 0.2991233117878437, "sampling/importance_sampling_ratio/min": 0.0021388806686445604, "sampling/sampling_logp_difference/max": 0.9958555936813355, "sampling/sampling_logp_difference/mean": 0.019412888586521147, "step": 50, "step_time": 45.46786805987358 }, { "clip_ratio/high_max": 0.0006968012428842485, "clip_ratio/high_mean": 0.0006968012428842485, "clip_ratio/low_mean": 0.00142822265625, "clip_ratio/low_min": 0.00142822265625, "clip_ratio/region_mean": 0.0021250238991342486, "completions/clipped_ratio": 0.5, "completions/max_length": 1923.8, "completions/max_terminated_length": 1387.0, "completions/mean_length": 1645.5875, "completions/mean_terminated_length": 1164.1375122070312, "completions/min_length": 1207.9, "completions/min_terminated_length": 1003.1, "entropy": 0.30428964346647264, "epoch": 0.08888888888888889, "frac_reward_zero_std": 0.65, "grad_norm": 0.09656701236963272, "learning_rate": 1.4532019704433498e-06, "loss": -0.013241247832775116, "num_tokens": 887087.0, "reward": 0.42500001564621925, "reward_std": 0.4399795174598694, "rewards/correctness_reward/mean": 0.375000000372529, "rewards/correctness_reward/std": 0.40771742463111876, "rewards/format_reward/mean": 0.05000000102445483, "rewards/format_reward/std": 0.033665436878800394, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0397414490580559, "sampling/importance_sampling_ratio/mean": 0.2622450416907668, "sampling/importance_sampling_ratio/min": 0.007209113225326291, "sampling/sampling_logp_difference/max": 0.8932502567768097, "sampling/sampling_logp_difference/mean": 0.01943899504840374, "step": 60, "step_time": 42.68353375019505 }, { "clip_ratio/high_max": 0.0005113344173878432, "clip_ratio/high_mean": 0.0005113344173878432, "clip_ratio/low_mean": 0.0008783510536886751, "clip_ratio/low_min": 0.0008783510536886751, "clip_ratio/region_mean": 0.0013896854710765184, "completions/clipped_ratio": 0.5375, "completions/max_length": 1989.6, "completions/max_terminated_length": 1319.0, "completions/mean_length": 1671.45, "completions/mean_terminated_length": 1026.7875, "completions/min_length": 1228.0, "completions/min_terminated_length": 818.4, "entropy": 0.29330393485724926, "epoch": 0.1037037037037037, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1.6995073891625615e-06, "loss": -0.005164426937699318, "num_tokens": 1033187.0, "reward": 0.30125001072883606, "reward_std": 0.43277348279953004, "rewards/correctness_reward/mean": 0.255000002682209, "rewards/correctness_reward/std": 0.4050062119960785, "rewards/format_reward/mean": 0.04625000059604645, "rewards/format_reward/std": 0.030469369143247604, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.1087342843413353, "sampling/importance_sampling_ratio/mean": 0.2937106598168612, "sampling/importance_sampling_ratio/min": 0.006787627846642863, "sampling/sampling_logp_difference/max": 0.7670994758605957, "sampling/sampling_logp_difference/mean": 0.019105390924960376, "step": 70, "step_time": 43.84505708152428 }, { "clip_ratio/high_max": 0.0004756878479383886, "clip_ratio/high_mean": 0.0004756878479383886, "clip_ratio/low_mean": 0.000604248046875, "clip_ratio/low_min": 0.000604248046875, "clip_ratio/region_mean": 0.0010799358948133885, "completions/clipped_ratio": 0.3375, "completions/max_length": 1897.5, "completions/max_terminated_length": 1425.4, "completions/mean_length": 1537.6625, "completions/mean_terminated_length": 1065.8619140625, "completions/min_length": 1209.8, "completions/min_terminated_length": 800.2, "entropy": 0.28736283620819447, "epoch": 0.11851851851851852, "frac_reward_zero_std": 0.8, "grad_norm": 0.052595239132642746, "learning_rate": 1.9458128078817736e-06, "loss": -0.019684906303882598, "num_tokens": 1167048.0, "reward": 0.6112500123679638, "reward_std": 0.2515163838863373, "rewards/correctness_reward/mean": 0.5450000029057265, "rewards/correctness_reward/std": 0.2362806797027588, "rewards/format_reward/mean": 0.06625000210478901, "rewards/format_reward/std": 0.015235702693462371, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6651582419872284, "sampling/importance_sampling_ratio/mean": 0.4043086588382721, "sampling/importance_sampling_ratio/min": 0.0058289245294872675, "sampling/sampling_logp_difference/max": 0.9541577339172364, "sampling/sampling_logp_difference/mean": 0.018320989795029163, "step": 80, "step_time": 41.49645183449611 }, { "clip_ratio/high_max": 0.00042016440420411527, "clip_ratio/high_mean": 0.00042016440420411527, "clip_ratio/low_mean": 0.0006747139064827934, "clip_ratio/low_min": 0.0006747139064827934, "clip_ratio/region_mean": 0.0010948783106869086, "completions/clipped_ratio": 0.4, "completions/max_length": 1899.9, "completions/max_terminated_length": 1522.2, "completions/mean_length": 1627.075, "completions/mean_terminated_length": 1263.7633422851563, "completions/min_length": 1264.8, "completions/min_terminated_length": 1060.0, "entropy": 0.3015484260395169, "epoch": 0.13333333333333333, "frac_reward_zero_std": 0.75, "grad_norm": 0.02362268790602684, "learning_rate": 2.1921182266009855e-06, "loss": -0.03768166303634644, "num_tokens": 1311750.0, "reward": 0.505000015348196, "reward_std": 0.26446898728609086, "rewards/correctness_reward/mean": 0.4450000002980232, "rewards/correctness_reward/std": 0.24084369465708733, "rewards/format_reward/mean": 0.06000000098720193, "rewards/format_reward/std": 0.02440723590552807, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 0.950841560959816, "sampling/importance_sampling_ratio/mean": 0.28259523659944535, "sampling/importance_sampling_ratio/min": 0.0034108405088772997, "sampling/sampling_logp_difference/max": 0.938099992275238, "sampling/sampling_logp_difference/mean": 0.01904191169887781, "step": 90, "step_time": 42.56532358508557 }, { "clip_ratio/high_max": 0.0008148401291691698, "clip_ratio/high_mean": 0.0008148401291691698, "clip_ratio/low_mean": 0.001190185546875, "clip_ratio/low_min": 0.001190185546875, "clip_ratio/region_mean": 0.00200502567604417, "completions/clipped_ratio": 0.55, "completions/max_length": 2048.0, "completions/max_terminated_length": 1450.6, "completions/mean_length": 1689.8125, "completions/mean_terminated_length": 1192.123828125, "completions/min_length": 1162.1, "completions/min_terminated_length": 957.3, "entropy": 0.2868456345051527, "epoch": 0.14814814814814814, "frac_reward_zero_std": 0.6, "grad_norm": 0.08160051703453064, "learning_rate": 2.4384236453201974e-06, "loss": -0.01632288545370102, "num_tokens": 1457479.0, "reward": 0.2726331319659948, "reward_std": 0.3918535321950912, "rewards/correctness_reward/mean": 0.22750000543892385, "rewards/correctness_reward/std": 0.3500069580972195, "rewards/format_reward/mean": 0.045000001695007084, "rewards/format_reward/std": 0.041792096197605134, "rewards/length_efficiency_reward/mean": 0.0001331249950453639, "rewards/length_efficiency_reward/std": 0.00025372845120728014, "sampling/importance_sampling_ratio/max": 1.074087956547737, "sampling/importance_sampling_ratio/mean": 0.2863946475088596, "sampling/importance_sampling_ratio/min": 0.010178243640621077, "sampling/sampling_logp_difference/max": 0.9397983431816102, "sampling/sampling_logp_difference/mean": 0.018534133210778236, "step": 100, "step_time": 44.84088846975938 }, { "clip_ratio/high_max": 0.0004754232446430251, "clip_ratio/high_mean": 0.0004754232446430251, "clip_ratio/low_mean": 0.001104736328125, "clip_ratio/low_min": 0.001104736328125, "clip_ratio/region_mean": 0.001580159572768025, "completions/clipped_ratio": 0.3, "completions/max_length": 1938.5, "completions/max_terminated_length": 1602.2, "completions/mean_length": 1459.025, "completions/mean_terminated_length": 1225.0010925292968, "completions/min_length": 978.4, "completions/min_terminated_length": 978.4, "entropy": 0.3161452116444707, "epoch": 0.16296296296296298, "frac_reward_zero_std": 0.7, "grad_norm": 0.09179597347974777, "learning_rate": 2.6847290640394093e-06, "loss": 0.038704752922058105, "num_tokens": 1584761.0, "reward": 0.7100000143051147, "reward_std": 0.4858191549777985, "rewards/correctness_reward/mean": 0.640000007301569, "rewards/correctness_reward/std": 0.4484484255313873, "rewards/format_reward/mean": 0.07000000197440386, "rewards/format_reward/std": 0.0373707041144371, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.439576455950737, "sampling/importance_sampling_ratio/mean": 0.33610708341002465, "sampling/importance_sampling_ratio/min": 0.007515860855346545, "sampling/sampling_logp_difference/max": 0.7350748300552368, "sampling/sampling_logp_difference/mean": 0.020000833831727505, "step": 110, "step_time": 42.10102230906487 }, { "clip_ratio/high_max": 0.00022550426656380295, "clip_ratio/high_mean": 0.00022550426656380295, "clip_ratio/low_mean": 0.000399233415373601, "clip_ratio/low_min": 0.000399233415373601, "clip_ratio/region_mean": 0.000624737681937404, "completions/clipped_ratio": 0.5375, "completions/max_length": 1995.2, "completions/max_terminated_length": 1063.3, "completions/mean_length": 1635.175, "completions/mean_terminated_length": 903.7150024414062, "completions/min_length": 1180.9, "completions/min_terminated_length": 771.3, "entropy": 0.2606943813152611, "epoch": 0.17777777777777778, "frac_reward_zero_std": 0.85, "grad_norm": 0.0, "learning_rate": 2.931034482758621e-06, "loss": -0.0008251294493675232, "num_tokens": 1729779.0, "reward": 0.3016600124537945, "reward_std": 0.37777974624186755, "rewards/correctness_reward/mean": 0.255000002682209, "rewards/correctness_reward/std": 0.3400575935840607, "rewards/format_reward/mean": 0.04625000059604645, "rewards/format_reward/std": 0.03724684417247772, "rewards/length_efficiency_reward/mean": 0.0004100000020116568, "rewards/length_efficiency_reward/std": 0.0005544389598071576, "sampling/importance_sampling_ratio/max": 1.2343256622552872, "sampling/importance_sampling_ratio/mean": 0.3580759346485138, "sampling/importance_sampling_ratio/min": 0.005624062876449898, "sampling/sampling_logp_difference/max": 0.8662369608879089, "sampling/sampling_logp_difference/mean": 0.017897653765976428, "step": 120, "step_time": 43.66495916871354 }, { "clip_ratio/high_max": 0.001094581844517961, "clip_ratio/high_mean": 0.001094581844517961, "clip_ratio/low_mean": 0.0014569461811333895, "clip_ratio/low_min": 0.0014569461811333895, "clip_ratio/region_mean": 0.002551528025651351, "completions/clipped_ratio": 0.35, "completions/max_length": 2019.7, "completions/max_terminated_length": 1637.1, "completions/mean_length": 1552.1875, "completions/mean_terminated_length": 1304.0478820800781, "completions/min_length": 1010.7, "completions/min_terminated_length": 1010.7, "entropy": 0.28869985416531563, "epoch": 0.1925925925925926, "frac_reward_zero_std": 0.5, "grad_norm": 0.9649873375892639, "learning_rate": 3.1773399014778327e-06, "loss": -0.09536395668983459, "num_tokens": 1865750.0, "reward": 0.49552564322948456, "reward_std": 0.5670612841844559, "rewards/correctness_reward/mean": 0.43000000417232515, "rewards/correctness_reward/std": 0.5315359912812709, "rewards/format_reward/mean": 0.06500000255182385, "rewards/format_reward/std": 0.03981267176568508, "rewards/length_efficiency_reward/mean": 0.0005256249918602407, "rewards/length_efficiency_reward/std": 0.0008113851014059037, "sampling/importance_sampling_ratio/max": 1.3021392822265625, "sampling/importance_sampling_ratio/mean": 0.3109331399202347, "sampling/importance_sampling_ratio/min": 0.0036396468727616592, "sampling/sampling_logp_difference/max": 1.0429420948028565, "sampling/sampling_logp_difference/mean": 0.01957107298076153, "step": 130, "step_time": 44.4100416389294 }, { "clip_ratio/high_max": 0.0005540439087781124, "clip_ratio/high_mean": 0.0005540439087781124, "clip_ratio/low_mean": 0.0019939872930990533, "clip_ratio/low_min": 0.0019939872930990533, "clip_ratio/region_mean": 0.0025480312018771657, "completions/clipped_ratio": 0.5, "completions/max_length": 2048.0, "completions/max_terminated_length": 1633.7, "completions/mean_length": 1676.1125, "completions/mean_terminated_length": 1357.2816772460938, "completions/min_length": 1121.8, "completions/min_terminated_length": 1121.8, "entropy": 0.29347491413354876, "epoch": 0.2074074074074074, "frac_reward_zero_std": 0.6, "grad_norm": 0.05394066870212555, "learning_rate": 3.423645320197045e-06, "loss": 0.03599870204925537, "num_tokens": 2009899.0, "reward": 0.3752200145274401, "reward_std": 0.5925859436392784, "rewards/correctness_reward/mean": 0.32500000335276125, "rewards/correctness_reward/std": 0.5436057150363922, "rewards/format_reward/mean": 0.05000000083819032, "rewards/format_reward/std": 0.05041723772883415, "rewards/length_efficiency_reward/mean": 0.00021999999880790712, "rewards/length_efficiency_reward/std": 0.0004386668093502522, "sampling/importance_sampling_ratio/max": 1.1674614436924458, "sampling/importance_sampling_ratio/mean": 0.2593226103112102, "sampling/importance_sampling_ratio/min": 0.0024672137107700108, "sampling/sampling_logp_difference/max": 0.8110510826110839, "sampling/sampling_logp_difference/mean": 0.019703016616404057, "step": 140, "step_time": 44.47214065417647 }, { "clip_ratio/high_max": 0.0007484834073693492, "clip_ratio/high_mean": 0.0007484834073693492, "clip_ratio/low_mean": 0.0015096336835995316, "clip_ratio/low_min": 0.0015096336835995316, "clip_ratio/region_mean": 0.002258117090968881, "completions/clipped_ratio": 0.5625, "completions/max_length": 2048.0, "completions/max_terminated_length": 1296.7, "completions/mean_length": 1731.0625, "completions/mean_terminated_length": 1114.9914489746093, "completions/min_length": 1365.0, "completions/min_terminated_length": 955.4, "entropy": 0.28509706994518635, "epoch": 0.2222222222222222, "frac_reward_zero_std": 0.6, "grad_norm": 0.0, "learning_rate": 3.6699507389162565e-06, "loss": 0.0106258362531662, "num_tokens": 2162320.0, "reward": 0.2812500067055225, "reward_std": 0.4311123609542847, "rewards/correctness_reward/mean": 0.23750000298023224, "rewards/correctness_reward/std": 0.39821332320570946, "rewards/format_reward/mean": 0.04375000158324838, "rewards/format_reward/std": 0.03699326254427433, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 0.9911381036043168, "sampling/importance_sampling_ratio/mean": 0.3147924717515707, "sampling/importance_sampling_ratio/min": 0.001622693703393452, "sampling/sampling_logp_difference/max": 0.7989792466163635, "sampling/sampling_logp_difference/mean": 0.018628334626555443, "step": 150, "step_time": 45.49694384559989 }, { "clip_ratio/high_max": 0.0006713618888170459, "clip_ratio/high_mean": 0.0006713618888170459, "clip_ratio/low_mean": 0.001323369934107177, "clip_ratio/low_min": 0.001323369934107177, "clip_ratio/region_mean": 0.0019947318229242227, "completions/clipped_ratio": 0.4375, "completions/max_length": 2048.0, "completions/max_terminated_length": 1689.2, "completions/mean_length": 1715.2625, "completions/mean_terminated_length": 1484.8262084960938, "completions/min_length": 1289.6, "completions/min_terminated_length": 1289.6, "entropy": 0.30604918748140336, "epoch": 0.23703703703703705, "frac_reward_zero_std": 0.65, "grad_norm": 0.2695084810256958, "learning_rate": 3.916256157635469e-06, "loss": -0.0020065195858478546, "num_tokens": 2309921.0, "reward": 0.41875001341104506, "reward_std": 0.5392595171928406, "rewards/correctness_reward/mean": 0.36250000037252905, "rewards/correctness_reward/std": 0.4975226067006588, "rewards/format_reward/mean": 0.05625000214204192, "rewards/format_reward/std": 0.045874805003404615, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.494359377026558, "sampling/importance_sampling_ratio/mean": 0.34138425141572953, "sampling/importance_sampling_ratio/min": 0.005483884311979636, "sampling/sampling_logp_difference/max": 0.8095157146453857, "sampling/sampling_logp_difference/mean": 0.019905944541096687, "step": 160, "step_time": 44.96130148917437 }, { "clip_ratio/high_max": 0.0009450995712541044, "clip_ratio/high_mean": 0.0009450995712541044, "clip_ratio/low_mean": 0.001202392578125, "clip_ratio/low_min": 0.001202392578125, "clip_ratio/region_mean": 0.0021474921493791045, "completions/clipped_ratio": 0.45, "completions/max_length": 2048.0, "completions/max_terminated_length": 1586.2, "completions/mean_length": 1659.8875, "completions/mean_terminated_length": 1328.0033569335938, "completions/min_length": 1115.7, "completions/min_terminated_length": 1115.7, "entropy": 0.3051243944093585, "epoch": 0.2518518518518518, "frac_reward_zero_std": 0.65, "grad_norm": 0.1204194575548172, "learning_rate": 4.16256157635468e-06, "loss": -0.034339231252670285, "num_tokens": 2454008.0, "reward": 0.4775000110268593, "reward_std": 0.6086134523153305, "rewards/correctness_reward/mean": 0.4225000083446503, "rewards/correctness_reward/std": 0.5618048131465911, "rewards/format_reward/mean": 0.05500000175088644, "rewards/format_reward/std": 0.048777279630303386, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2148158952593804, "sampling/importance_sampling_ratio/mean": 0.23397584930062293, "sampling/importance_sampling_ratio/min": 0.001304598978094873, "sampling/sampling_logp_difference/max": 0.7190336763858796, "sampling/sampling_logp_difference/mean": 0.01982088889926672, "step": 170, "step_time": 44.95190164670348 }, { "clip_ratio/high_max": 0.000854317846824415, "clip_ratio/high_mean": 0.000854317846824415, "clip_ratio/low_mean": 0.002571762620937079, "clip_ratio/low_min": 0.002571762620937079, "clip_ratio/region_mean": 0.0034260804677614942, "completions/clipped_ratio": 0.5375, "completions/max_length": 1968.3, "completions/max_terminated_length": 1760.0, "completions/mean_length": 1759.4625, "completions/mean_terminated_length": 1549.3150146484375, "completions/min_length": 1362.0, "completions/min_terminated_length": 1362.0, "entropy": 0.29519203063100574, "epoch": 0.26666666666666666, "frac_reward_zero_std": 0.45, "grad_norm": 0.08190698176622391, "learning_rate": 4.408866995073892e-06, "loss": -0.050768601894378665, "num_tokens": 2607341.0, "reward": 0.23875001072883606, "reward_std": 0.33369648903608323, "rewards/correctness_reward/mean": 0.1924999963492155, "rewards/correctness_reward/std": 0.2971598096191883, "rewards/format_reward/mean": 0.046250001061707735, "rewards/format_reward/std": 0.040190113335847856, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 0.8491888105869293, "sampling/importance_sampling_ratio/mean": 0.23999512735754253, "sampling/importance_sampling_ratio/min": 0.004403389227809385, "sampling/sampling_logp_difference/max": 0.9504605531692505, "sampling/sampling_logp_difference/mean": 0.019836217164993286, "step": 180, "step_time": 43.68827957296744 }, { "clip_ratio/high_max": 0.0007269132765941322, "clip_ratio/high_mean": 0.0007269132765941322, "clip_ratio/low_mean": 0.0011277107463683934, "clip_ratio/low_min": 0.0011277107463683934, "clip_ratio/region_mean": 0.0018546240229625256, "completions/clipped_ratio": 0.35, "completions/max_length": 2037.1, "completions/max_terminated_length": 1436.1, "completions/mean_length": 1428.15, "completions/mean_terminated_length": 961.245849609375, "completions/min_length": 910.1, "completions/min_terminated_length": 705.3, "entropy": 0.290473675634712, "epoch": 0.2814814814814815, "frac_reward_zero_std": 0.65, "grad_norm": 0.0, "learning_rate": 4.655172413793104e-06, "loss": -0.05426536202430725, "num_tokens": 2734121.0, "reward": 0.5834337756037712, "reward_std": 0.4872488170862198, "rewards/correctness_reward/mean": 0.5174999967217445, "rewards/correctness_reward/std": 0.4543666332960129, "rewards/format_reward/mean": 0.06500000171363354, "rewards/format_reward/std": 0.03455207794904709, "rewards/length_efficiency_reward/mean": 0.0009337499737739563, "rewards/length_efficiency_reward/std": 0.0010841149836778642, "sampling/importance_sampling_ratio/max": 1.2027183800935746, "sampling/importance_sampling_ratio/mean": 0.37490637227892876, "sampling/importance_sampling_ratio/min": 0.002881228062324226, "sampling/sampling_logp_difference/max": 0.7551185309886932, "sampling/sampling_logp_difference/mean": 0.020146298967301846, "step": 190, "step_time": 44.321867388300596 }, { "clip_ratio/high_max": 0.0009448761833482422, "clip_ratio/high_mean": 0.0009448761833482422, "clip_ratio/low_mean": 0.00203857421875, "clip_ratio/low_min": 0.00203857421875, "clip_ratio/region_mean": 0.002983450402098242, "completions/clipped_ratio": 0.375, "completions/max_length": 1983.9, "completions/max_terminated_length": 1808.7, "completions/mean_length": 1673.125, "completions/mean_terminated_length": 1495.26181640625, "completions/min_length": 1236.2, "completions/min_terminated_length": 1236.2, "entropy": 0.29930071607232095, "epoch": 0.2962962962962963, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 4.901477832512316e-06, "loss": 0.00047026127576828004, "num_tokens": 2879135.0, "reward": 0.4875000099651515, "reward_std": 0.4853708490729332, "rewards/correctness_reward/mean": 0.42500000484287737, "rewards/correctness_reward/std": 0.44509575963020326, "rewards/format_reward/mean": 0.06250000111758709, "rewards/format_reward/std": 0.043300296366214755, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4676499903202056, "sampling/importance_sampling_ratio/mean": 0.3165957175195217, "sampling/importance_sampling_ratio/min": 0.0016230770910624416, "sampling/sampling_logp_difference/max": 0.8405740261077881, "sampling/sampling_logp_difference/mean": 0.019820176251232623, "step": 200, "step_time": 43.806333302240816 }, { "clip_ratio/high_max": 0.0010216034715995193, "clip_ratio/high_mean": 0.0010216034715995193, "clip_ratio/low_mean": 0.001767312770243734, "clip_ratio/low_min": 0.001767312770243734, "clip_ratio/region_mean": 0.0027889162418432535, "completions/clipped_ratio": 0.3, "completions/max_length": 1987.0, "completions/max_terminated_length": 1798.4, "completions/mean_length": 1603.05, "completions/mean_terminated_length": 1418.630126953125, "completions/min_length": 1099.1, "completions/min_terminated_length": 1099.1, "entropy": 0.3053171422332525, "epoch": 0.3111111111111111, "frac_reward_zero_std": 0.5, "grad_norm": 0.1703852117061615, "learning_rate": 4.999969989929363e-06, "loss": -0.017523324489593504, "num_tokens": 3020395.0, "reward": 0.5062500156462193, "reward_std": 0.5737184286117554, "rewards/correctness_reward/mean": 0.4375000029802322, "rewards/correctness_reward/std": 0.5355811148881913, "rewards/format_reward/mean": 0.06875000223517418, "rewards/format_reward/std": 0.042885662615299226, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2357708513736725, "sampling/importance_sampling_ratio/mean": 0.3290738731622696, "sampling/importance_sampling_ratio/min": 0.005334117155871354, "sampling/sampling_logp_difference/max": 0.8891814708709717, "sampling/sampling_logp_difference/mean": 0.020076879672706126, "step": 210, "step_time": 43.872724731639025 }, { "clip_ratio/high_max": 0.0014869271457428113, "clip_ratio/high_mean": 0.0014869271457428113, "clip_ratio/low_mean": 0.002307795366505161, "clip_ratio/low_min": 0.002307795366505161, "clip_ratio/region_mean": 0.0037947225122479723, "completions/clipped_ratio": 0.275, "completions/max_length": 1920.1, "completions/max_terminated_length": 1726.1, "completions/mean_length": 1536.6375, "completions/mean_terminated_length": 1397.8363525390625, "completions/min_length": 1119.2, "completions/min_terminated_length": 1119.2, "entropy": 0.29589710254222157, "epoch": 0.32592592592592595, "frac_reward_zero_std": 0.35, "grad_norm": 0.06872892379760742, "learning_rate": 4.999786597662395e-06, "loss": 0.02164815068244934, "num_tokens": 3152882.0, "reward": 0.6050756372511387, "reward_std": 0.3723608944565058, "rewards/correctness_reward/mean": 0.5325000017881394, "rewards/correctness_reward/std": 0.34375100657343866, "rewards/format_reward/mean": 0.07250000322237611, "rewards/format_reward/std": 0.032827488332986834, "rewards/length_efficiency_reward/mean": 7.562499813502654e-05, "rewards/length_efficiency_reward/std": 0.00019279402331449092, "sampling/importance_sampling_ratio/max": 1.4064665712416171, "sampling/importance_sampling_ratio/mean": 0.3650544978678226, "sampling/importance_sampling_ratio/min": 0.0016844410914927722, "sampling/sampling_logp_difference/max": 0.9379635095596314, "sampling/sampling_logp_difference/mean": 0.02021803893148899, "step": 220, "step_time": 42.302905085124074 }, { "clip_ratio/high_max": 0.0009918318093696144, "clip_ratio/high_mean": 0.0009918318093696144, "clip_ratio/low_mean": 0.0020838231779634953, "clip_ratio/low_min": 0.0020838231779634953, "clip_ratio/region_mean": 0.0030756549873331095, "completions/clipped_ratio": 0.375, "completions/max_length": 1920.2, "completions/max_terminated_length": 1705.6, "completions/mean_length": 1551.2375, "completions/mean_terminated_length": 1366.9947875976563, "completions/min_length": 1094.3, "completions/min_terminated_length": 1094.3, "entropy": 0.288944486528635, "epoch": 0.34074074074074073, "frac_reward_zero_std": 0.55, "grad_norm": 0.0063933017663657665, "learning_rate": 4.999436497605313e-06, "loss": -0.05875176787376404, "num_tokens": 3288209.0, "reward": 0.4250000136904418, "reward_std": 0.4044877901673317, "rewards/correctness_reward/mean": 0.3624999985098839, "rewards/correctness_reward/std": 0.3707671508193016, "rewards/format_reward/mean": 0.06250000139698386, "rewards/format_reward/std": 0.03703123778104782, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3248377382755279, "sampling/importance_sampling_ratio/mean": 0.32935924381017684, "sampling/importance_sampling_ratio/min": 0.013732147745395195, "sampling/sampling_logp_difference/max": 0.7944218158721924, "sampling/sampling_logp_difference/mean": 0.020012675505131482, "step": 230, "step_time": 42.160917016025635 }, { "clip_ratio/high_max": 0.0015348217522841879, "clip_ratio/high_mean": 0.0015348217522841879, "clip_ratio/low_mean": 0.001543489083996974, "clip_ratio/low_min": 0.001543489083996974, "clip_ratio/region_mean": 0.003078310836281162, "completions/clipped_ratio": 0.425, "completions/max_length": 1995.1, "completions/max_terminated_length": 1639.8, "completions/mean_length": 1654.35, "completions/mean_terminated_length": 1302.2677490234375, "completions/min_length": 1253.0, "completions/min_terminated_length": 1048.2, "entropy": 0.2760451960377395, "epoch": 0.35555555555555557, "frac_reward_zero_std": 0.45, "grad_norm": 0.18857723474502563, "learning_rate": 4.998919713105873e-06, "loss": 0.045951706171035764, "num_tokens": 3432501.0, "reward": 0.43518938943743707, "reward_std": 0.4700739842606708, "rewards/correctness_reward/mean": 0.3775000009685755, "rewards/correctness_reward/std": 0.4404321610927582, "rewards/format_reward/mean": 0.05750000206753612, "rewards/format_reward/std": 0.03501462116837502, "rewards/length_efficiency_reward/mean": 0.0001893749984446913, "rewards/length_efficiency_reward/std": 0.000535633391700685, "sampling/importance_sampling_ratio/max": 1.191299396380782, "sampling/importance_sampling_ratio/mean": 0.23651103908196092, "sampling/importance_sampling_ratio/min": 0.0007759356201859191, "sampling/sampling_logp_difference/max": 1.0235774338245391, "sampling/sampling_logp_difference/mean": 0.019437621906399728, "step": 240, "step_time": 44.25449943756685 }, { "clip_ratio/high_max": 0.0004865065508056432, "clip_ratio/high_mean": 0.0004865065508056432, "clip_ratio/low_mean": 0.001269581902306527, "clip_ratio/low_min": 0.001269581902306527, "clip_ratio/region_mean": 0.00175608845311217, "completions/clipped_ratio": 0.3375, "completions/max_length": 1965.4, "completions/max_terminated_length": 1423.8, "completions/mean_length": 1478.125, "completions/mean_terminated_length": 1030.8341735839845, "completions/min_length": 940.3, "completions/min_terminated_length": 735.5, "entropy": 0.27253339197486637, "epoch": 0.37037037037037035, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 4.998236278627815e-06, "loss": 0.013929113745689392, "num_tokens": 3561195.0, "reward": 0.5991250202059746, "reward_std": 0.44888501763343813, "rewards/correctness_reward/mean": 0.5325000002980232, "rewards/correctness_reward/std": 0.4187964558601379, "rewards/format_reward/mean": 0.0662500012665987, "rewards/format_reward/std": 0.030469369143247604, "rewards/length_efficiency_reward/mean": 0.0003749999916180968, "rewards/length_efficiency_reward/std": 0.0007201722357422114, "sampling/importance_sampling_ratio/max": 1.4570940136909485, "sampling/importance_sampling_ratio/mean": 0.36110749691724775, "sampling/importance_sampling_ratio/min": 0.0036850016433163544, "sampling/sampling_logp_difference/max": 1.0234840512275696, "sampling/sampling_logp_difference/mean": 0.0196872366592288, "step": 250, "step_time": 42.70053608352318 }, { "clip_ratio/high_max": 0.0006014656843035482, "clip_ratio/high_mean": 0.0006014656843035482, "clip_ratio/low_mean": 0.0010760711156763136, "clip_ratio/low_min": 0.0010760711156763136, "clip_ratio/region_mean": 0.0016775367999798617, "completions/clipped_ratio": 0.3375, "completions/max_length": 1920.7, "completions/max_terminated_length": 1525.4, "completions/mean_length": 1555.85, "completions/mean_terminated_length": 1295.61083984375, "completions/min_length": 1088.3, "completions/min_terminated_length": 1088.3, "entropy": 0.30174778401851654, "epoch": 0.3851851851851852, "frac_reward_zero_std": 0.7, "grad_norm": 0.0, "learning_rate": 4.997386239748572e-06, "loss": -0.008143197745084763, "num_tokens": 3695239.0, "reward": 0.5612500183284282, "reward_std": 0.5392331033945084, "rewards/correctness_reward/mean": 0.49500000290572643, "rewards/correctness_reward/std": 0.5068671077489852, "rewards/format_reward/mean": 0.06625000098720193, "rewards/format_reward/std": 0.03455129489302635, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3191817864775657, "sampling/importance_sampling_ratio/mean": 0.3094415869563818, "sampling/importance_sampling_ratio/min": 0.0006343921588268131, "sampling/sampling_logp_difference/max": 0.9605819165706635, "sampling/sampling_logp_difference/mean": 0.02142522968351841, "step": 260, "step_time": 41.96377791473642 }, { "clip_ratio/high_max": 0.0006801372495829128, "clip_ratio/high_mean": 0.0006801372495829128, "clip_ratio/low_mean": 0.001143087330274284, "clip_ratio/low_min": 0.001143087330274284, "clip_ratio/region_mean": 0.0018232245798571966, "completions/clipped_ratio": 0.1875, "completions/max_length": 1879.8, "completions/max_terminated_length": 1790.1, "completions/mean_length": 1481.5125, "completions/mean_terminated_length": 1373.0877685546875, "completions/min_length": 1058.7, "completions/min_terminated_length": 1058.7, "entropy": 0.29690768755972385, "epoch": 0.4, "frac_reward_zero_std": 0.65, "grad_norm": 0.0, "learning_rate": 4.996369653156224e-06, "loss": 0.020374581217765808, "num_tokens": 3823592.0, "reward": 0.7812500178813935, "reward_std": 0.37789265513420106, "rewards/correctness_reward/mean": 0.7000000029802322, "rewards/correctness_reward/std": 0.35426762104034426, "rewards/format_reward/mean": 0.08125000223517417, "rewards/format_reward/std": 0.027566111460328103, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6554171323776246, "sampling/importance_sampling_ratio/mean": 0.3656636871397495, "sampling/importance_sampling_ratio/min": 0.005212117801420391, "sampling/sampling_logp_difference/max": 1.0982315421104432, "sampling/sampling_logp_difference/mean": 0.020670222491025923, "step": 270, "step_time": 40.92184403641149 }, { "clip_ratio/high_max": 0.0007834594405721873, "clip_ratio/high_mean": 0.0007834594405721873, "clip_ratio/low_mean": 0.0010274637839756905, "clip_ratio/low_min": 0.0010274637839756905, "clip_ratio/region_mean": 0.001810923224547878, "completions/clipped_ratio": 0.475, "completions/max_length": 2009.1, "completions/max_terminated_length": 1274.6, "completions/mean_length": 1662.2, "completions/mean_terminated_length": 1059.1476440429688, "completions/min_length": 1236.2, "completions/min_terminated_length": 826.6, "entropy": 0.27445439994335175, "epoch": 0.4148148148148148, "frac_reward_zero_std": 0.7, "grad_norm": 0.0, "learning_rate": 4.99518658664572e-06, "loss": 0.015945900976657868, "num_tokens": 3968992.0, "reward": 0.37064126078039406, "reward_std": 0.39439683854579927, "rewards/correctness_reward/mean": 0.31750000230968, "rewards/correctness_reward/std": 0.3617725603282452, "rewards/format_reward/mean": 0.05250000171363354, "rewards/format_reward/std": 0.03400490321218967, "rewards/length_efficiency_reward/mean": 0.0006412499584257603, "rewards/length_efficiency_reward/std": 0.0011599787045270204, "sampling/importance_sampling_ratio/max": 1.2043530493974686, "sampling/importance_sampling_ratio/mean": 0.2813464432954788, "sampling/importance_sampling_ratio/min": 0.004266520892269909, "sampling/sampling_logp_difference/max": 0.8425451517105103, "sampling/sampling_logp_difference/mean": 0.019642751850187778, "step": 280, "step_time": 44.50383766526356 }, { "clip_ratio/high_max": 0.000732948406948708, "clip_ratio/high_mean": 0.000732948406948708, "clip_ratio/low_mean": 0.0018018761125858872, "clip_ratio/low_min": 0.0018018761125858872, "clip_ratio/region_mean": 0.0025348245195345952, "completions/clipped_ratio": 0.3875, "completions/max_length": 1968.9, "completions/max_terminated_length": 1757.6, "completions/mean_length": 1589.7375, "completions/mean_terminated_length": 1380.3553649902344, "completions/min_length": 1052.8, "completions/min_terminated_length": 1052.8, "entropy": 0.3148526353761554, "epoch": 0.42962962962962964, "frac_reward_zero_std": 0.6, "grad_norm": 0.09379521757364273, "learning_rate": 4.9938371191143545e-06, "loss": -0.006783177703619003, "num_tokens": 4107243.0, "reward": 0.35948626659810545, "reward_std": 0.23933057636022567, "rewards/correctness_reward/mean": 0.2975000008940697, "rewards/correctness_reward/std": 0.21100954562425614, "rewards/format_reward/mean": 0.061250001192092896, "rewards/format_reward/std": 0.03093191236257553, "rewards/length_efficiency_reward/mean": 0.0007362500167801045, "rewards/length_efficiency_reward/std": 0.001169990870403126, "sampling/importance_sampling_ratio/max": 1.0605366311967372, "sampling/importance_sampling_ratio/mean": 0.2691107377409935, "sampling/importance_sampling_ratio/min": 0.0024449194475892, "sampling/sampling_logp_difference/max": 0.8307014465332031, "sampling/sampling_logp_difference/mean": 0.02143716476857662, "step": 290, "step_time": 43.142127541638914 }, { "clip_ratio/high_max": 0.0009672275089542381, "clip_ratio/high_mean": 0.0009672275089542381, "clip_ratio/low_mean": 0.0019725356076378377, "clip_ratio/low_min": 0.0019725356076378377, "clip_ratio/region_mean": 0.0029397631165920756, "completions/clipped_ratio": 0.2375, "completions/max_length": 1867.2, "completions/max_terminated_length": 1566.9, "completions/mean_length": 1423.55, "completions/mean_terminated_length": 1146.6309692382813, "completions/min_length": 1001.1, "completions/min_terminated_length": 796.3, "entropy": 0.282509620860219, "epoch": 0.4444444444444444, "frac_reward_zero_std": 0.55, "grad_norm": 0.27018725872039795, "learning_rate": 4.992321340556511e-06, "loss": 0.023980888724327087, "num_tokens": 4233779.0, "reward": 0.6047487691044807, "reward_std": 0.3638819932937622, "rewards/correctness_reward/mean": 0.5275000028312207, "rewards/correctness_reward/std": 0.3479676455259323, "rewards/format_reward/mean": 0.07625000216066838, "rewards/format_reward/std": 0.01759178563952446, "rewards/length_efficiency_reward/mean": 0.0009987500496208667, "rewards/length_efficiency_reward/std": 0.0019052334129810334, "sampling/importance_sampling_ratio/max": 0.9052107781171799, "sampling/importance_sampling_ratio/mean": 0.25276198983192444, "sampling/importance_sampling_ratio/min": 0.003962857421356603, "sampling/sampling_logp_difference/max": 1.1523001730442046, "sampling/sampling_logp_difference/mean": 0.02052048034965992, "step": 300, "step_time": 40.97691578529775 }, { "clip_ratio/high_max": 0.001341156986018177, "clip_ratio/high_mean": 0.001341156986018177, "clip_ratio/low_mean": 0.001668647344922647, "clip_ratio/low_min": 0.001668647344922647, "clip_ratio/region_mean": 0.003009804330940824, "completions/clipped_ratio": 0.4, "completions/max_length": 1953.1, "completions/max_terminated_length": 1532.0, "completions/mean_length": 1639.225, "completions/mean_terminated_length": 1299.2888305664062, "completions/min_length": 1224.6, "completions/min_terminated_length": 1019.8, "entropy": 0.27802186692133546, "epoch": 0.45925925925925926, "frac_reward_zero_std": 0.45, "grad_norm": 0.0519985631108284, "learning_rate": 4.9906393520576556e-06, "loss": -0.012239690870046616, "num_tokens": 4375625.0, "reward": 0.4060575166717172, "reward_std": 0.397552827000618, "rewards/correctness_reward/mean": 0.3450000025331974, "rewards/correctness_reward/std": 0.3672131113708019, "rewards/format_reward/mean": 0.060000001732259986, "rewards/format_reward/std": 0.03202547878026962, "rewards/length_efficiency_reward/mean": 0.0010575000080280006, "rewards/length_efficiency_reward/std": 0.0019632406532764436, "sampling/importance_sampling_ratio/max": 0.772921297699213, "sampling/importance_sampling_ratio/mean": 0.21906356066465377, "sampling/importance_sampling_ratio/min": 0.0044086123671149835, "sampling/sampling_logp_difference/max": 0.8263131678104401, "sampling/sampling_logp_difference/mean": 0.019935403019189835, "step": 310, "step_time": 43.0815763713792 }, { "clip_ratio/high_max": 0.0006290169083513319, "clip_ratio/high_mean": 0.0006290169083513319, "clip_ratio/low_mean": 0.0019635351607576014, "clip_ratio/low_min": 0.0019635351607576014, "clip_ratio/region_mean": 0.002592552069108933, "completions/clipped_ratio": 0.3, "completions/max_length": 1833.8, "completions/max_terminated_length": 1578.9, "completions/mean_length": 1403.5, "completions/mean_terminated_length": 1204.8208374023438, "completions/min_length": 944.1, "completions/min_terminated_length": 944.1, "entropy": 0.2959835690446198, "epoch": 0.4740740740740741, "frac_reward_zero_std": 0.6, "grad_norm": 0.0050934613682329655, "learning_rate": 4.988791265787598e-06, "loss": -0.028436988592147827, "num_tokens": 4497385.0, "reward": 0.5976912699639797, "reward_std": 0.5006729900138452, "rewards/correctness_reward/mean": 0.5275000024586916, "rewards/correctness_reward/std": 0.46703119575977325, "rewards/format_reward/mean": 0.07000000104308128, "rewards/format_reward/std": 0.036360986158251764, "rewards/length_efficiency_reward/mean": 0.00019125000399071722, "rewards/length_efficiency_reward/std": 0.0003576488350518048, "sampling/importance_sampling_ratio/max": 1.6210725069046021, "sampling/importance_sampling_ratio/mean": 0.3590504204854369, "sampling/importance_sampling_ratio/min": 0.010417400204460137, "sampling/sampling_logp_difference/max": 0.9554403066635132, "sampling/sampling_logp_difference/mean": 0.021173207182437183, "step": 320, "step_time": 39.995778077933934 }, { "clip_ratio/high_max": 0.0007380636860034428, "clip_ratio/high_mean": 0.0007380636860034428, "clip_ratio/low_mean": 0.0015335470001446084, "clip_ratio/low_min": 0.0015335470001446084, "clip_ratio/region_mean": 0.002271610686148051, "completions/clipped_ratio": 0.35, "completions/max_length": 1967.2, "completions/max_terminated_length": 1526.8, "completions/mean_length": 1542.9125, "completions/mean_terminated_length": 1142.6, "completions/min_length": 1039.1, "completions/min_terminated_length": 834.3, "entropy": 0.2726613922044635, "epoch": 0.4888888888888889, "frac_reward_zero_std": 0.6, "grad_norm": 0.19719338417053223, "learning_rate": 4.986777204993008e-06, "loss": -0.02510994076728821, "num_tokens": 4631894.0, "reward": 0.49543002583086493, "reward_std": 0.37422274476848544, "rewards/correctness_reward/mean": 0.4299999985843897, "rewards/correctness_reward/std": 0.346181583404541, "rewards/format_reward/mean": 0.06500000134110451, "rewards/format_reward/std": 0.030639102309942247, "rewards/length_efficiency_reward/mean": 0.00043000001169275494, "rewards/length_efficiency_reward/std": 0.0012162237311713398, "sampling/importance_sampling_ratio/max": 1.2216215431690216, "sampling/importance_sampling_ratio/mean": 0.28127982951700686, "sampling/importance_sampling_ratio/min": 0.006505116555490531, "sampling/sampling_logp_difference/max": 1.0323942065238954, "sampling/sampling_logp_difference/mean": 0.02025210000574589, "step": 330, "step_time": 43.185658453404905 }, { "clip_ratio/high_max": 0.0007842323451768607, "clip_ratio/high_mean": 0.0007842323451768607, "clip_ratio/low_mean": 0.0013084923848509789, "clip_ratio/low_min": 0.0013084923848509789, "clip_ratio/region_mean": 0.0020927247300278396, "completions/clipped_ratio": 0.3, "completions/max_length": 1834.3, "completions/max_terminated_length": 1683.2, "completions/mean_length": 1481.1625, "completions/mean_terminated_length": 1299.4791870117188, "completions/min_length": 1004.9, "completions/min_terminated_length": 1004.9, "entropy": 0.3037892831489444, "epoch": 0.5037037037037037, "frac_reward_zero_std": 0.6, "grad_norm": 0.036585550755262375, "learning_rate": 4.984597303989203e-06, "loss": -0.035512736439704894, "num_tokens": 4761135.0, "reward": 0.5725000180304051, "reward_std": 0.373321944475174, "rewards/correctness_reward/mean": 0.5024999983608722, "rewards/correctness_reward/std": 0.35145958289504053, "rewards/format_reward/mean": 0.07000000141561032, "rewards/format_reward/std": 0.024238285794854163, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0646295830607415, "sampling/importance_sampling_ratio/mean": 0.25783716291189196, "sampling/importance_sampling_ratio/min": 0.0009405114920809865, "sampling/sampling_logp_difference/max": 0.9845355749130249, "sampling/sampling_logp_difference/mean": 0.0206430546939373, "step": 340, "step_time": 40.29749016510323 }, { "clip_ratio/high_max": 0.00105124544352293, "clip_ratio/high_mean": 0.00105124544352293, "clip_ratio/low_mean": 0.0017492531653260813, "clip_ratio/low_min": 0.0017492531653260813, "clip_ratio/region_mean": 0.0028004986088490114, "completions/clipped_ratio": 0.325, "completions/max_length": 1783.0, "completions/max_terminated_length": 1493.0, "completions/mean_length": 1435.4, "completions/mean_terminated_length": 1113.8075073242187, "completions/min_length": 1041.7, "completions/min_terminated_length": 836.9, "entropy": 0.28469121772795913, "epoch": 0.5185185185185185, "frac_reward_zero_std": 0.5, "grad_norm": 0.13037614524364471, "learning_rate": 4.982251708151182e-06, "loss": -0.008781444281339645, "num_tokens": 4889547.0, "reward": 0.6028200149536133, "reward_std": 0.3006966353626922, "rewards/correctness_reward/mean": 0.5350000008940696, "rewards/correctness_reward/std": 0.2793744206428528, "rewards/format_reward/mean": 0.06750000100582838, "rewards/format_reward/std": 0.02495441064238548, "rewards/length_efficiency_reward/mean": 0.00032000000355765225, "rewards/length_efficiency_reward/std": 0.000829992606304586, "sampling/importance_sampling_ratio/max": 1.2942826308310031, "sampling/importance_sampling_ratio/mean": 0.3292302343994379, "sampling/importance_sampling_ratio/min": 0.009803920293052215, "sampling/sampling_logp_difference/max": 1.0382127165794373, "sampling/sampling_logp_difference/mean": 0.021288632042706014, "step": 350, "step_time": 39.56919716345146 }, { "clip_ratio/high_max": 0.000776523316744715, "clip_ratio/high_mean": 0.000776523316744715, "clip_ratio/low_mean": 0.0008377842605113983, "clip_ratio/low_min": 0.0008377842605113983, "clip_ratio/region_mean": 0.0016143075772561133, "completions/clipped_ratio": 0.25, "completions/max_length": 1892.5, "completions/max_terminated_length": 1640.6, "completions/mean_length": 1489.1875, "completions/mean_terminated_length": 1356.7735961914063, "completions/min_length": 1100.6, "completions/min_terminated_length": 1100.6, "entropy": 0.29693945199251176, "epoch": 0.5333333333333333, "frac_reward_zero_std": 0.7, "grad_norm": 0.027122685685753822, "learning_rate": 4.979740573903939e-06, "loss": -0.013264276087284088, "num_tokens": 5021542.0, "reward": 0.612500012665987, "reward_std": 0.4079078257083893, "rewards/correctness_reward/mean": 0.5375000067055226, "rewards/correctness_reward/std": 0.3805114448070526, "rewards/format_reward/mean": 0.07500000204890966, "rewards/format_reward/std": 0.02739637829363346, "rewards/length_efficiency_reward/mean": 0.0, "rewards/length_efficiency_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0460569851100445, "sampling/importance_sampling_ratio/mean": 0.2716627847403288, "sampling/importance_sampling_ratio/min": 0.0013100236756145023, "sampling/sampling_logp_difference/max": 1.0467463374137878, "sampling/sampling_logp_difference/mean": 0.021687238477170467, "step": 360, "step_time": 41.54590162793174 }, { "clip_ratio/high_max": 0.00041593528148951007, "clip_ratio/high_mean": 0.00041593528148951007, "clip_ratio/low_mean": 0.0015723522898042574, "clip_ratio/low_min": 0.0015723522898042574, "clip_ratio/region_mean": 0.0019882875712937675, "completions/clipped_ratio": 0.1, "completions/max_length": 1640.0, "completions/max_terminated_length": 1491.5, "completions/mean_length": 1174.6375, "completions/mean_terminated_length": 1078.0750122070312, "completions/min_length": 728.0, "completions/min_terminated_length": 728.0, "entropy": 0.2599758941680193, "epoch": 0.5481481481481482, "frac_reward_zero_std": 0.7, "grad_norm": 0.0, "learning_rate": 4.9770640687120225e-06, "loss": 4.998829099349678e-05, "num_tokens": 5125417.0, "reward": 0.8706750243902206, "reward_std": 0.29858288893010465, "rewards/correctness_reward/mean": 0.7799999982118606, "rewards/correctness_reward/std": 0.28387848734855653, "rewards/format_reward/mean": 0.09000000171363354, "rewards/format_reward/std": 0.014603426307439804, "rewards/length_efficiency_reward/mean": 0.0006750000058673323, "rewards/length_efficiency_reward/std": 0.0009173117810860276, "sampling/importance_sampling_ratio/max": 1.612634265422821, "sampling/importance_sampling_ratio/mean": 0.4575915545225143, "sampling/importance_sampling_ratio/min": 0.024248149202321656, "sampling/sampling_logp_difference/max": 1.3486943364143371, "sampling/sampling_logp_difference/mean": 0.02114765774458647, "step": 370, "step_time": 35.599055078905074 }, { "clip_ratio/high_max": 0.0013537989667383953, "clip_ratio/high_mean": 0.0013537989667383953, "clip_ratio/low_mean": 0.0023303459282033147, "clip_ratio/low_min": 0.0023303459282033147, "clip_ratio/region_mean": 0.00368414489494171, "completions/clipped_ratio": 0.3375, "completions/max_length": 1948.7, "completions/max_terminated_length": 1461.9, "completions/mean_length": 1489.75, "completions/mean_terminated_length": 1186.797509765625, "completions/min_length": 936.8, "completions/min_terminated_length": 936.8, "entropy": 0.24692531703040005, "epoch": 0.562962962962963, "frac_reward_zero_std": 0.5, "grad_norm": 0.47570788860321045, "learning_rate": 4.974222371068377e-06, "loss": -0.017648613452911376, "num_tokens": 5256529.0, "reward": 0.47563501000404357, "reward_std": 0.5546515717636794, "rewards/correctness_reward/mean": 0.40750000476837156, "rewards/correctness_reward/std": 0.5171626001596451, "rewards/format_reward/mean": 0.06625000089406967, "rewards/format_reward/std": 0.0415364783257246, "rewards/length_efficiency_reward/mean": 0.0018850000575184821, "rewards/length_efficiency_reward/std": 0.0027629806194454432, "sampling/importance_sampling_ratio/max": 1.3096152998507022, "sampling/importance_sampling_ratio/mean": 0.3175370816141367, "sampling/importance_sampling_ratio/min": 0.0023430767469108105, "sampling/sampling_logp_difference/max": 1.3119081497192382, "sampling/sampling_logp_difference/mean": 0.019368173833936454, "step": 380, "step_time": 42.530021231994034 }, { "clip_ratio/high_max": 0.00011172954982612282, "clip_ratio/high_mean": 0.00011172954982612282, "clip_ratio/low_mean": 0.000498578860424459, "clip_ratio/low_min": 0.000498578860424459, "clip_ratio/region_mean": 0.0006103084102505818, "completions/clipped_ratio": 0.1875, "completions/max_length": 1876.5, "completions/max_terminated_length": 1591.7, "completions/mean_length": 1335.6875, "completions/mean_terminated_length": 1132.2549987792968, "completions/min_length": 798.3, "completions/min_terminated_length": 798.3, "entropy": 0.25275117214769127, "epoch": 0.5777777777777777, "frac_reward_zero_std": 0.9, "grad_norm": 0.0, "learning_rate": 4.971215670482433e-06, "loss": 0.007638859748840332, "num_tokens": 5377020.0, "reward": 0.7444250226020813, "reward_std": 0.3823041975498199, "rewards/correctness_reward/mean": 0.6625000029802323, "rewards/correctness_reward/std": 0.3611502945423126, "rewards/format_reward/mean": 0.0812500011175871, "rewards/format_reward/std": 0.02121116816997528, "rewards/length_efficiency_reward/mean": 0.0006749999709427356, "rewards/length_efficiency_reward/std": 0.000975111685693264, "sampling/importance_sampling_ratio/max": 1.352328333258629, "sampling/importance_sampling_ratio/mean": 0.39203007221221925, "sampling/importance_sampling_ratio/min": 0.024190397560596467, "sampling/sampling_logp_difference/max": 1.1286945700645448, "sampling/sampling_logp_difference/mean": 0.018978366162627935, "step": 390, "step_time": 41.28239038791507 }, { "clip_ratio/high_max": 0.0009029380613355898, "clip_ratio/high_mean": 0.0009029380613355898, "clip_ratio/low_mean": 0.0021462386124767364, "clip_ratio/low_min": 0.0021462386124767364, "clip_ratio/region_mean": 0.003049176673812326, "completions/clipped_ratio": 0.4, "completions/max_length": 1902.9, "completions/max_terminated_length": 1442.9, "completions/mean_length": 1509.9125, "completions/mean_terminated_length": 1112.8726318359375, "completions/min_length": 1067.8, "completions/min_terminated_length": 863.0, "entropy": 0.2667001824826002, "epoch": 0.5925925925925926, "frac_reward_zero_std": 0.55, "grad_norm": 0.0, "learning_rate": 4.968044167467471e-06, "loss": -0.0237930566072464, "num_tokens": 5509989.0, "reward": 0.44328813403844836, "reward_std": 0.22918477602215717, "rewards/correctness_reward/mean": 0.38250000178813937, "rewards/correctness_reward/std": 0.20425259992480277, "rewards/format_reward/mean": 0.06000000135973096, "rewards/format_reward/std": 0.02811250314116478, "rewards/length_efficiency_reward/mean": 0.0007881250043283216, "rewards/length_efficiency_reward/std": 0.0013779858811176382, "sampling/importance_sampling_ratio/max": 0.9269534617662429, "sampling/importance_sampling_ratio/mean": 0.26210661344230174, "sampling/importance_sampling_ratio/min": 0.013762131108160247, "sampling/sampling_logp_difference/max": 1.090614414215088, "sampling/sampling_logp_difference/mean": 0.021061623468995094, "step": 400, "step_time": 42.2569040347822 }, { "clip_ratio/high_max": 0.0013804055663058535, "clip_ratio/high_mean": 0.0013804055663058535, "clip_ratio/low_mean": 0.0012150652532000096, "clip_ratio/low_min": 0.0012150652532000096, "clip_ratio/region_mean": 0.002595470819505863, "completions/clipped_ratio": 0.3375, "completions/max_length": 1993.5, "completions/max_terminated_length": 1623.1, "completions/mean_length": 1524.55, "completions/mean_terminated_length": 1270.602392578125, "completions/min_length": 956.5, "completions/min_terminated_length": 956.5, "entropy": 0.2511401602067053, "epoch": 0.6074074074074074, "frac_reward_zero_std": 0.6, "grad_norm": 0.08325648307800293, "learning_rate": 4.964708073527247e-06, "loss": 0.04337623119354248, "num_tokens": 5647141.0, "reward": 0.4745043819770217, "reward_std": 0.4443462465889752, "rewards/correctness_reward/mean": 0.4075000014156103, "rewards/correctness_reward/std": 0.40372326895594596, "rewards/format_reward/mean": 0.06625000219792128, "rewards/format_reward/std": 0.04271671250462532, "rewards/length_efficiency_reward/mean": 0.0007543750107288361, "rewards/length_efficiency_reward/std": 0.001600624481216073, "sampling/importance_sampling_ratio/max": 1.218729043379426, "sampling/importance_sampling_ratio/mean": 0.27182662757113574, "sampling/importance_sampling_ratio/min": 0.0007386480399873107, "sampling/sampling_logp_difference/max": 0.9590512037277221, "sampling/sampling_logp_difference/mean": 0.01950228279456496, "step": 410, "step_time": 44.20556942569092 }, { "clip_ratio/high_max": 0.000520808677538298, "clip_ratio/high_mean": 0.000520808677538298, "clip_ratio/low_mean": 0.002329328324412927, "clip_ratio/low_min": 0.002329328324412927, "clip_ratio/region_mean": 0.002850137001951225, "completions/clipped_ratio": 0.2875, "completions/max_length": 1846.0, "completions/max_terminated_length": 1586.5, "completions/mean_length": 1330.7625, "completions/mean_terminated_length": 1061.3091735839844, "completions/min_length": 719.3, "completions/min_terminated_length": 719.3, "entropy": 0.2457987274043262, "epoch": 0.6222222222222222, "frac_reward_zero_std": 0.55, "grad_norm": 0.00043478753650560975, "learning_rate": 4.961207611141892e-06, "loss": 0.04510357081890106, "num_tokens": 5764682.0, "reward": 0.6770918995141983, "reward_std": 0.443152716755867, "rewards/correctness_reward/mean": 0.605000001564622, "rewards/correctness_reward/std": 0.4107246309518814, "rewards/format_reward/mean": 0.07125000115484, "rewards/format_reward/std": 0.034759003296494485, "rewards/length_efficiency_reward/mean": 0.0008418749552220106, "rewards/length_efficiency_reward/std": 0.0017472409177571534, "sampling/importance_sampling_ratio/max": 1.170586159825325, "sampling/importance_sampling_ratio/mean": 0.2847073331475258, "sampling/importance_sampling_ratio/min": 0.00627761494833976, "sampling/sampling_logp_difference/max": 1.0515904188156129, "sampling/sampling_logp_difference/mean": 0.019648726005107163, "step": 420, "step_time": 15.205995783116668 }, { "clip_ratio/high_max": 0.0012077969979145565, "clip_ratio/high_mean": 0.0012077969979145565, "clip_ratio/low_mean": 0.0020778057805728167, "clip_ratio/low_min": 0.0020778057805728167, "clip_ratio/region_mean": 0.003285602778487373, "completions/clipped_ratio": 0.2375, "completions/max_length": 1741.9, "completions/max_terminated_length": 1569.6, "completions/mean_length": 1289.3125, "completions/mean_terminated_length": 1176.6214538574218, "completions/min_length": 901.8, "completions/min_terminated_length": 901.8, "entropy": 0.25596501845866443, "epoch": 0.6370370370370371, "frac_reward_zero_std": 0.5, "grad_norm": 0.0735541358590126, "learning_rate": 4.957543013753074e-06, "loss": 0.03737008571624756, "num_tokens": 5879691.0, "reward": 0.6420612819492817, "reward_std": 0.34433289982844145, "rewards/correctness_reward/mean": 0.5650000005960465, "rewards/correctness_reward/std": 0.32224635109305383, "rewards/format_reward/mean": 0.07625000225380063, "rewards/format_reward/std": 0.02558668702840805, "rewards/length_efficiency_reward/mean": 0.000811250033439137, "rewards/length_efficiency_reward/std": 0.0013803998474031687, "sampling/importance_sampling_ratio/max": 0.9550634175539017, "sampling/importance_sampling_ratio/mean": 0.21740142367780207, "sampling/importance_sampling_ratio/min": 0.002628398803062737, "sampling/sampling_logp_difference/max": 1.2587344706058503, "sampling/sampling_logp_difference/mean": 0.021320880204439164, "step": 430, "step_time": 13.235985807236284 }, { "clip_ratio/high_max": 0.001160441932734102, "clip_ratio/high_mean": 0.001160441932734102, "clip_ratio/low_mean": 0.0018974495425936766, "clip_ratio/low_min": 0.0018974495425936766, "clip_ratio/region_mean": 0.0030578914753277785, "completions/clipped_ratio": 0.1875, "completions/max_length": 1810.1, "completions/max_terminated_length": 1448.3, "completions/mean_length": 1322.7, "completions/mean_terminated_length": 1067.583935546875, "completions/min_length": 980.3, "completions/min_terminated_length": 775.5, "entropy": 0.2797438757494092, "epoch": 0.6518518518518519, "frac_reward_zero_std": 0.5, "grad_norm": 0.033258359879255295, "learning_rate": 4.953714525748425e-06, "loss": -0.0144338920712471, "num_tokens": 5997439.0, "reward": 0.6201724942773581, "reward_std": 0.3690946189686656, "rewards/correctness_reward/mean": 0.5375000067055226, "rewards/correctness_reward/std": 0.3538807094097137, "rewards/format_reward/mean": 0.08125000260770321, "rewards/format_reward/std": 0.01595182754099369, "rewards/length_efficiency_reward/mean": 0.0014224999926227611, "rewards/length_efficiency_reward/std": 0.001877962778962683, "sampling/importance_sampling_ratio/max": 1.16160203255713, "sampling/importance_sampling_ratio/mean": 0.26618634946644304, "sampling/importance_sampling_ratio/min": 0.0044567355536855756, "sampling/sampling_logp_difference/max": 1.4923224329948426, "sampling/sampling_logp_difference/mean": 0.023057046346366405, "step": 440, "step_time": 13.52226556390524 }, { "clip_ratio/high_max": 0.001066775037907064, "clip_ratio/high_mean": 0.001066775037907064, "clip_ratio/low_mean": 0.0031727533118100838, "clip_ratio/low_min": 0.0031727533118100838, "clip_ratio/region_mean": 0.004239528349717148, "completions/clipped_ratio": 0.2875, "completions/max_length": 1963.3, "completions/max_terminated_length": 1728.3, "completions/mean_length": 1492.425, "completions/mean_terminated_length": 1310.9959716796875, "completions/min_length": 986.8, "completions/min_terminated_length": 986.8, "entropy": 0.2799477934837341, "epoch": 0.6666666666666666, "frac_reward_zero_std": 0.45, "grad_norm": 0.11226266622543335, "learning_rate": 4.9497224024452505e-06, "loss": -0.018950110673904418, "num_tokens": 6130113.0, "reward": 0.5514043845236302, "reward_std": 0.5257057053851895, "rewards/correctness_reward/mean": 0.4800000060349703, "rewards/correctness_reward/std": 0.4949623137712479, "rewards/format_reward/mean": 0.07125000208616257, "rewards/format_reward/std": 0.03825656212866306, "rewards/length_efficiency_reward/mean": 0.0001543750026030466, "rewards/length_efficiency_reward/std": 0.0004366384469904006, "sampling/importance_sampling_ratio/max": 1.1865878872573377, "sampling/importance_sampling_ratio/mean": 0.24820118136703967, "sampling/importance_sampling_ratio/min": 0.0021500205621123314, "sampling/sampling_logp_difference/max": 1.0478538513183593, "sampling/sampling_logp_difference/mean": 0.022717707604169846, "step": 450, "step_time": 14.479338533803821 }, { "clip_ratio/high_max": 0.000974494856200181, "clip_ratio/high_mean": 0.000974494856200181, "clip_ratio/low_mean": 0.0015656882984330877, "clip_ratio/low_min": 0.0015656882984330877, "clip_ratio/region_mean": 0.0025401831546332685, "completions/clipped_ratio": 0.1125, "completions/max_length": 1639.9, "completions/max_terminated_length": 1495.2, "completions/mean_length": 1187.3375, "completions/mean_terminated_length": 1099.9994140625, "completions/min_length": 764.5, "completions/min_terminated_length": 764.5, "entropy": 0.2678541589528322, "epoch": 0.6814814814814815, "frac_reward_zero_std": 0.55, "grad_norm": 0.008453172631561756, "learning_rate": 4.945566910073496e-06, "loss": -0.0022131526842713357, "num_tokens": 6234956.0, "reward": 0.5912668995559216, "reward_std": 0.42797074615955355, "rewards/correctness_reward/mean": 0.5025000013411045, "rewards/correctness_reward/std": 0.41150325536727905, "rewards/format_reward/mean": 0.0887500025331974, "rewards/format_reward/std": 0.021504761278629304, "rewards/length_efficiency_reward/mean": 1.687500043772161e-05, "rewards/length_efficiency_reward/std": 4.772970860358328e-05, "sampling/importance_sampling_ratio/max": 0.937442260235548, "sampling/importance_sampling_ratio/mean": 0.2380099237896502, "sampling/importance_sampling_ratio/min": 0.004414617783913855, "sampling/sampling_logp_difference/max": 1.2503409028053283, "sampling/sampling_logp_difference/mean": 0.024316846579313278, "step": 460, "step_time": 12.337617406714708 }, { "clip_ratio/high_max": 0.0005923955701291561, "clip_ratio/high_mean": 0.0005923955701291561, "clip_ratio/low_mean": 0.0010009765625, "clip_ratio/low_min": 0.0010009765625, "clip_ratio/region_mean": 0.0015933721326291562, "completions/clipped_ratio": 0.35, "completions/max_length": 1922.0, "completions/max_terminated_length": 1658.8, "completions/mean_length": 1503.875, "completions/mean_terminated_length": 1291.8682250976562, "completions/min_length": 977.7, "completions/min_terminated_length": 977.7, "entropy": 0.27749953884631395, "epoch": 0.6962962962962963, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 4.941248325757999e-06, "loss": 0.019719459116458893, "num_tokens": 6369490.0, "reward": 0.4952112632803619, "reward_std": 0.30235492140054704, "rewards/correctness_reward/mean": 0.4299999997019768, "rewards/correctness_reward/std": 0.27131965607404707, "rewards/format_reward/mean": 0.06500000124797226, "rewards/format_reward/std": 0.032741603627800944, "rewards/length_efficiency_reward/mean": 0.00021124999038875104, "rewards/length_efficiency_reward/std": 0.0004160163924098015, "sampling/importance_sampling_ratio/max": 0.9228326313197612, "sampling/importance_sampling_ratio/mean": 0.20876362472772597, "sampling/importance_sampling_ratio/min": 0.0018225521551357815, "sampling/sampling_logp_difference/max": 1.2317867398262023, "sampling/sampling_logp_difference/mean": 0.022405552119016646, "step": 470, "step_time": 13.98218732206151 }, { "clip_ratio/high_max": 0.0015531498545897193, "clip_ratio/high_mean": 0.0015531498545897193, "clip_ratio/low_mean": 0.002355109012569301, "clip_ratio/low_min": 0.002355109012569301, "clip_ratio/region_mean": 0.00390825886715902, "completions/clipped_ratio": 0.2, "completions/max_length": 1922.6, "completions/max_terminated_length": 1767.7, "completions/mean_length": 1363.8375, "completions/mean_terminated_length": 1207.8476440429688, "completions/min_length": 833.8, "completions/min_terminated_length": 833.8, "entropy": 0.2549503302201629, "epoch": 0.7111111111111111, "frac_reward_zero_std": 0.4, "grad_norm": 0.17366230487823486, "learning_rate": 4.9367669375000016e-06, "loss": -0.00034297243691980837, "num_tokens": 6490117.0, "reward": 0.5759268904104828, "reward_std": 0.46614947337657214, "rewards/correctness_reward/mean": 0.4949999999254942, "rewards/correctness_reward/std": 0.4405492417514324, "rewards/format_reward/mean": 0.07875000201165676, "rewards/format_reward/std": 0.028489944711327552, "rewards/length_efficiency_reward/mean": 0.002176875034228942, "rewards/length_efficiency_reward/std": 0.003052720451159985, "sampling/importance_sampling_ratio/max": 0.5789117172360421, "sampling/importance_sampling_ratio/mean": 0.16507304161787034, "sampling/importance_sampling_ratio/min": 0.0008232366482843645, "sampling/sampling_logp_difference/max": 1.4114261627197267, "sampling/sampling_logp_difference/mean": 0.022164187394082546, "step": 480, "step_time": 13.961506398115308 }, { "clip_ratio/high_max": 0.0012949575466336682, "clip_ratio/high_mean": 0.0012949575466336682, "clip_ratio/low_mean": 0.002268781268503517, "clip_ratio/low_min": 0.002268781268503517, "clip_ratio/region_mean": 0.003563738815137185, "completions/clipped_ratio": 0.3, "completions/max_length": 1850.5, "completions/max_terminated_length": 1461.2, "completions/mean_length": 1368.525, "completions/mean_terminated_length": 1091.098486328125, "completions/min_length": 830.6, "completions/min_terminated_length": 830.6, "entropy": 0.2822195779532194, "epoch": 0.725925925925926, "frac_reward_zero_std": 0.55, "grad_norm": 0.03088097646832466, "learning_rate": 4.932123044157948e-06, "loss": -0.01430053561925888, "num_tokens": 6610863.0, "reward": 0.5737125113606453, "reward_std": 0.47703770995140077, "rewards/correctness_reward/mean": 0.5025000050663948, "rewards/correctness_reward/std": 0.4438177704811096, "rewards/format_reward/mean": 0.07000000216066837, "rewards/format_reward/std": 0.038086828961968425, "rewards/length_efficiency_reward/mean": 0.0012125000677770004, "rewards/length_efficiency_reward/std": 0.001666794065386057, "sampling/importance_sampling_ratio/max": 0.7662897758185864, "sampling/importance_sampling_ratio/mean": 0.1852167235687375, "sampling/importance_sampling_ratio/min": 0.0002008963233947725, "sampling/sampling_logp_difference/max": 1.7439379334449767, "sampling/sampling_logp_difference/mean": 0.023953136056661606, "step": 490, "step_time": 13.586232338286937 }, { "clip_ratio/high_max": 0.0007055908063193783, "clip_ratio/high_mean": 0.0007055908063193783, "clip_ratio/low_mean": 0.0017364088096655905, "clip_ratio/low_min": 0.0017364088096655905, "clip_ratio/region_mean": 0.002441999615984969, "completions/clipped_ratio": 0.2375, "completions/max_length": 1697.6, "completions/max_terminated_length": 1382.1, "completions/mean_length": 1263.3625, "completions/mean_terminated_length": 956.5791870117188, "completions/min_length": 809.9, "completions/min_terminated_length": 605.1, "entropy": 0.2561162358149886, "epoch": 0.7407407407407407, "frac_reward_zero_std": 0.6, "grad_norm": 0.0, "learning_rate": 4.927316955427553e-06, "loss": 0.009614716470241546, "num_tokens": 6723624.0, "reward": 0.6011493846774101, "reward_std": 0.40467396974563596, "rewards/correctness_reward/mean": 0.5250000014901162, "rewards/correctness_reward/std": 0.3874266743659973, "rewards/format_reward/mean": 0.07500000186264515, "rewards/format_reward/std": 0.01868535205721855, "rewards/length_efficiency_reward/mean": 0.0011493750145746163, "rewards/length_efficiency_reward/std": 0.001929981915600365, "sampling/importance_sampling_ratio/max": 1.0601293578743936, "sampling/importance_sampling_ratio/mean": 0.2344768911600113, "sampling/importance_sampling_ratio/min": 0.00406664201291278, "sampling/sampling_logp_difference/max": 1.3843498706817627, "sampling/sampling_logp_difference/mean": 0.022158562485128642, "step": 500, "step_time": 12.639872438274324 }, { "clip_ratio/high_max": 0.0010111481184139847, "clip_ratio/high_mean": 0.0010111481184139847, "clip_ratio/low_mean": 0.0014403547917027026, "clip_ratio/low_min": 0.0014403547917027026, "clip_ratio/region_mean": 0.0024515029101166872, "completions/clipped_ratio": 0.4, "completions/max_length": 1907.8, "completions/max_terminated_length": 1311.0, "completions/mean_length": 1541.1125, "completions/mean_terminated_length": 994.9968017578125, "completions/min_length": 1171.6, "completions/min_terminated_length": 762.0, "entropy": 0.282977326028049, "epoch": 0.7555555555555555, "frac_reward_zero_std": 0.65, "grad_norm": 0.0, "learning_rate": 4.9223489918211474e-06, "loss": -0.02192380130290985, "num_tokens": 6863769.0, "reward": 0.43037001639604566, "reward_std": 0.3525068175280467, "rewards/correctness_reward/mean": 0.3700000043958426, "rewards/correctness_reward/std": 0.3327335864305496, "rewards/format_reward/mean": 0.0600000012665987, "rewards/format_reward/std": 0.02474670223891735, "rewards/length_efficiency_reward/mean": 0.0003700000001117587, "rewards/length_efficiency_reward/std": 0.0008033483289182186, "sampling/importance_sampling_ratio/max": 0.7351960234344006, "sampling/importance_sampling_ratio/mean": 0.1641315458342433, "sampling/importance_sampling_ratio/min": 0.001335060031420663, "sampling/sampling_logp_difference/max": 3.0151263236999513, "sampling/sampling_logp_difference/mean": 0.022794026974588634, "step": 510, "step_time": 14.159395257197321 }, { "clip_ratio/high_max": 0.0017303690110566095, "clip_ratio/high_mean": 0.0017303690110566095, "clip_ratio/low_mean": 0.0019024976223590783, "clip_ratio/low_min": 0.0019024976223590783, "clip_ratio/region_mean": 0.0036328666334156877, "completions/clipped_ratio": 0.275, "completions/max_length": 1831.2, "completions/max_terminated_length": 1549.8, "completions/mean_length": 1401.9, "completions/mean_terminated_length": 1222.0272827148438, "completions/min_length": 928.1, "completions/min_terminated_length": 928.1, "entropy": 0.2969978377223015, "epoch": 0.7703703703703704, "frac_reward_zero_std": 0.45, "grad_norm": 0.0035613523796200752, "learning_rate": 4.917219484646307e-06, "loss": 0.012761344015598298, "num_tokens": 6987389.0, "reward": 0.5428349981084466, "reward_std": 0.4179523440077901, "rewards/correctness_reward/mean": 0.4700000058859587, "rewards/correctness_reward/std": 0.3853103265166283, "rewards/format_reward/mean": 0.0725000025704503, "rewards/format_reward/std": 0.03573074601590633, "rewards/length_efficiency_reward/mean": 0.00033500001300126316, "rewards/length_efficiency_reward/std": 0.0008980812504887581, "sampling/importance_sampling_ratio/max": 0.8631399072706699, "sampling/importance_sampling_ratio/mean": 0.24986709821969272, "sampling/importance_sampling_ratio/min": 0.015631077784928494, "sampling/sampling_logp_difference/max": 1.092229437828064, "sampling/sampling_logp_difference/mean": 0.024158950708806515, "step": 520, "step_time": 13.532434344943614 }, { "clip_ratio/high_max": 0.0009452382597373799, "clip_ratio/high_mean": 0.0009452382597373799, "clip_ratio/low_mean": 0.0015841079817619174, "clip_ratio/low_min": 0.0015841079817619174, "clip_ratio/region_mean": 0.002529346241499297, "completions/clipped_ratio": 0.375, "completions/max_length": 1968.6, "completions/max_terminated_length": 1560.3, "completions/mean_length": 1569.8375, "completions/mean_terminated_length": 1197.125, "completions/min_length": 1065.9, "completions/min_terminated_length": 861.1, "entropy": 0.26213434133678676, "epoch": 0.7851851851851852, "frac_reward_zero_std": 0.6, "grad_norm": 0.0, "learning_rate": 4.9119287759837545e-06, "loss": 0.020042675733566283, "num_tokens": 7126536.0, "reward": 0.45057750791311263, "reward_std": 0.41371997983660547, "rewards/correctness_reward/mean": 0.38750000223517417, "rewards/correctness_reward/std": 0.38816905915737154, "rewards/format_reward/mean": 0.06250000130385161, "rewards/format_reward/std": 0.029922977462410925, "rewards/length_efficiency_reward/mean": 0.0005775000099674799, "rewards/length_efficiency_reward/std": 0.001141538099909667, "sampling/importance_sampling_ratio/max": 1.170878465473652, "sampling/importance_sampling_ratio/mean": 0.2623008359223604, "sampling/importance_sampling_ratio/min": 0.0010534422388445818, "sampling/sampling_logp_difference/max": 1.6341172695159911, "sampling/sampling_logp_difference/mean": 0.02109828917309642, "step": 530, "step_time": 14.59560510609299 }, { "clip_ratio/high_max": 0.000902684053289704, "clip_ratio/high_mean": 0.000902684053289704, "clip_ratio/low_mean": 0.001709118764847517, "clip_ratio/low_min": 0.001709118764847517, "clip_ratio/region_mean": 0.002611802818137221, "completions/clipped_ratio": 0.25, "completions/max_length": 1695.7, "completions/max_terminated_length": 1314.4, "completions/mean_length": 1183.0625, "completions/mean_terminated_length": 926.6625, "completions/min_length": 687.0, "completions/min_terminated_length": 687.0, "entropy": 0.2682175524532795, "epoch": 0.8, "frac_reward_zero_std": 0.6, "grad_norm": 0.0, "learning_rate": 4.906477218664546e-06, "loss": -0.07770953178405762, "num_tokens": 7233609.0, "reward": 0.615944380685687, "reward_std": 0.4304513517767191, "rewards/correctness_reward/mean": 0.5375, "rewards/correctness_reward/std": 0.4015724301338196, "rewards/format_reward/mean": 0.07500000130385161, "rewards/format_reward/std": 0.025293876975774766, "rewards/length_efficiency_reward/mean": 0.0034443750279024242, "rewards/length_efficiency_reward/std": 0.004157532565295696, "sampling/importance_sampling_ratio/max": 1.5010828226804733, "sampling/importance_sampling_ratio/mean": 0.3413222625851631, "sampling/importance_sampling_ratio/min": 0.007177348095137859, "sampling/sampling_logp_difference/max": 1.2137579023838043, "sampling/sampling_logp_difference/mean": 0.02223182376474142, "step": 540, "step_time": 12.800919637829065 }, { "clip_ratio/high_max": 0.0025106097251409664, "clip_ratio/high_mean": 0.0025106097251409664, "clip_ratio/low_mean": 0.0027347358700353652, "clip_ratio/low_min": 0.0027347358700353652, "clip_ratio/region_mean": 0.005245345595176332, "completions/clipped_ratio": 0.2, "completions/max_length": 1891.1, "completions/max_terminated_length": 1658.0, "completions/mean_length": 1286.9875, "completions/mean_terminated_length": 1121.5818115234374, "completions/min_length": 737.6, "completions/min_terminated_length": 737.6, "entropy": 0.26885042143985627, "epoch": 0.8148148148148148, "frac_reward_zero_std": 0.25, "grad_norm": 0.08994826674461365, "learning_rate": 4.900865176246546e-06, "loss": 0.0004219723865389824, "num_tokens": 7346888.0, "reward": 0.7056412637233734, "reward_std": 0.520521582220681, "rewards/correctness_reward/mean": 0.6225000053644181, "rewards/correctness_reward/std": 0.4894020676612854, "rewards/format_reward/mean": 0.08000000342726707, "rewards/format_reward/std": 0.033751321583986284, "rewards/length_efficiency_reward/mean": 0.003141249978216365, "rewards/length_efficiency_reward/std": 0.004340173932723701, "sampling/importance_sampling_ratio/max": 0.7490350410342217, "sampling/importance_sampling_ratio/mean": 0.15175360813736916, "sampling/importance_sampling_ratio/min": 0.00197476179964724, "sampling/sampling_logp_difference/max": 1.7838127255439757, "sampling/sampling_logp_difference/mean": 0.024590420722961425, "step": 550, "step_time": 14.069852022733539 }, { "clip_ratio/high_max": 0.0009343817240733187, "clip_ratio/high_mean": 0.0009343817240733187, "clip_ratio/low_mean": 0.0027257578680291773, "clip_ratio/low_min": 0.0027257578680291773, "clip_ratio/region_mean": 0.003660139592102496, "completions/clipped_ratio": 0.225, "completions/max_length": 1851.7, "completions/max_terminated_length": 1473.7, "completions/mean_length": 1331.9625, "completions/mean_terminated_length": 1124.8916870117187, "completions/min_length": 815.3, "completions/min_terminated_length": 815.3, "entropy": 0.27642152886837723, "epoch": 0.8296296296296296, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 4.895093022990176e-06, "loss": -0.032569941878318784, "num_tokens": 7469193.0, "reward": 0.6087712597101926, "reward_std": 0.3964171044994146, "rewards/correctness_reward/mean": 0.5300000037997961, "rewards/correctness_reward/std": 0.372175395488739, "rewards/format_reward/mean": 0.07750000227242708, "rewards/format_reward/std": 0.027019719779491424, "rewards/length_efficiency_reward/mean": 0.0012712499708868563, "rewards/length_efficiency_reward/std": 0.0023299440974369643, "sampling/importance_sampling_ratio/max": 0.8284099403768778, "sampling/importance_sampling_ratio/mean": 0.16827086750417947, "sampling/importance_sampling_ratio/min": 0.0012340840625256534, "sampling/sampling_logp_difference/max": 1.9904000878334045, "sampling/sampling_logp_difference/mean": 0.024701233580708505, "step": 560, "step_time": 13.999212111718952 }, { "clip_ratio/high_max": 0.0012827725295210257, "clip_ratio/high_mean": 0.0012827725295210257, "clip_ratio/low_mean": 0.0029375463112955914, "clip_ratio/low_min": 0.0029375463112955914, "clip_ratio/region_mean": 0.004220318840816617, "completions/clipped_ratio": 0.4, "completions/max_length": 1991.1, "completions/max_terminated_length": 1167.1, "completions/mean_length": 1388.9375, "completions/mean_terminated_length": 823.6272705078125, "completions/min_length": 752.4, "completions/min_terminated_length": 547.6, "entropy": 0.2753100197762251, "epoch": 0.8444444444444444, "frac_reward_zero_std": 0.5, "grad_norm": 0.07388363778591156, "learning_rate": 4.889161143833461e-06, "loss": -0.004594495519995689, "num_tokens": 7594940.0, "reward": 0.2945931423455477, "reward_std": 0.4456210508942604, "rewards/correctness_reward/mean": 0.23250000029802323, "rewards/correctness_reward/std": 0.4098650962114334, "rewards/format_reward/mean": 0.060000001639127734, "rewards/format_reward/std": 0.03935012854635715, "rewards/length_efficiency_reward/mean": 0.0020931249833665787, "rewards/length_efficiency_reward/std": 0.0028115474968217315, "sampling/importance_sampling_ratio/max": 1.2905457910150289, "sampling/importance_sampling_ratio/mean": 0.27891073962673546, "sampling/importance_sampling_ratio/min": 0.001791392979794182, "sampling/sampling_logp_difference/max": 2.35616819858551, "sampling/sampling_logp_difference/mean": 0.02452029138803482, "step": 570, "step_time": 14.720530469343066 }, { "clip_ratio/high_max": 0.0010564374621026217, "clip_ratio/high_mean": 0.0010564374621026217, "clip_ratio/low_mean": 0.002178305416600779, "clip_ratio/low_min": 0.002178305416600779, "clip_ratio/region_mean": 0.0032347428787034005, "completions/clipped_ratio": 0.2375, "completions/max_length": 1837.6, "completions/max_terminated_length": 1486.5, "completions/mean_length": 1245.1375, "completions/mean_terminated_length": 993.0524963378906, "completions/min_length": 668.7, "completions/min_terminated_length": 668.7, "entropy": 0.25112604154273865, "epoch": 0.8592592592592593, "frac_reward_zero_std": 0.5, "grad_norm": 0.22860242426395416, "learning_rate": 4.883069934366355e-06, "loss": -0.007899226248264312, "num_tokens": 7706283.0, "reward": 0.6297806516289711, "reward_std": 0.45098284266423433, "rewards/correctness_reward/mean": 0.5525000046938657, "rewards/correctness_reward/std": 0.4246478736400604, "rewards/format_reward/mean": 0.07625000141561031, "rewards/format_reward/std": 0.02992219440639019, "rewards/length_efficiency_reward/mean": 0.0010306249809218571, "rewards/length_efficiency_reward/std": 0.002080874651437625, "sampling/importance_sampling_ratio/max": 1.2887410774827004, "sampling/importance_sampling_ratio/mean": 0.27414565458893775, "sampling/importance_sampling_ratio/min": 0.00303689510874392, "sampling/sampling_logp_difference/max": 1.4570694327354432, "sampling/sampling_logp_difference/mean": 0.022690335661172865, "step": 580, "step_time": 13.722587811108678 }, { "clip_ratio/high_max": 0.0024907320927013642, "clip_ratio/high_mean": 0.0024907320927013642, "clip_ratio/low_mean": 0.0035540570461307652, "clip_ratio/low_min": 0.0035540570461307652, "clip_ratio/region_mean": 0.00604478913883213, "completions/clipped_ratio": 0.225, "completions/max_length": 1628.6, "completions/max_terminated_length": 1534.3, "completions/mean_length": 1267.4875, "completions/mean_terminated_length": 1162.3883422851563, "completions/min_length": 919.3, "completions/min_terminated_length": 919.3, "entropy": 0.25036735478788613, "epoch": 0.8740740740740741, "frac_reward_zero_std": 0.25, "grad_norm": 0.21488115191459656, "learning_rate": 4.876819800804361e-06, "loss": 0.01002853587269783, "num_tokens": 7819794.0, "reward": 0.7093925148248672, "reward_std": 0.39697957527823746, "rewards/correctness_reward/mean": 0.630000001937151, "rewards/correctness_reward/std": 0.3752748340368271, "rewards/format_reward/mean": 0.07750000115483999, "rewards/format_reward/std": 0.020325310155749322, "rewards/length_efficiency_reward/mean": 0.0018925000447779893, "rewards/length_efficiency_reward/std": 0.0026768976589664818, "sampling/importance_sampling_ratio/max": 1.127157373726368, "sampling/importance_sampling_ratio/mean": 0.23004143834114074, "sampling/importance_sampling_ratio/min": 0.0011307188771752408, "sampling/sampling_logp_difference/max": 2.0075063943862914, "sampling/sampling_logp_difference/mean": 0.02614522036164999, "step": 590, "step_time": 12.918895542621613 }, { "clip_ratio/high_max": 0.0012476142233936115, "clip_ratio/high_mean": 0.0012476142233936115, "clip_ratio/low_mean": 0.0029561413801275195, "clip_ratio/low_min": 0.0029561413801275195, "clip_ratio/region_mean": 0.004203755603521131, "completions/clipped_ratio": 0.3125, "completions/max_length": 1723.1, "completions/max_terminated_length": 1367.0, "completions/mean_length": 1329.7875, "completions/mean_terminated_length": 954.3520324707031, "completions/min_length": 847.8, "completions/min_terminated_length": 643.0, "entropy": 0.23313154662027955, "epoch": 0.8888888888888888, "frac_reward_zero_std": 0.5, "grad_norm": 0.02515222877264023, "learning_rate": 4.87041115996144e-06, "loss": -0.016968238353729247, "num_tokens": 7939653.0, "reward": 0.46963064186275005, "reward_std": 0.40666334112174807, "rewards/correctness_reward/mean": 0.40000000074505804, "rewards/correctness_reward/std": 0.38490029275417326, "rewards/format_reward/mean": 0.06875000149011612, "rewards/format_reward/std": 0.02457696907222271, "rewards/length_efficiency_reward/mean": 0.0008806250087218359, "rewards/length_efficiency_reward/std": 0.0018143961497116834, "sampling/importance_sampling_ratio/max": 0.7667963312007486, "sampling/importance_sampling_ratio/mean": 0.18017131788656116, "sampling/importance_sampling_ratio/min": 0.0015338808225351386, "sampling/sampling_logp_difference/max": 2.6043832421302797, "sampling/sampling_logp_difference/mean": 0.025299737229943275, "step": 600, "step_time": 13.424872031342238 }, { "clip_ratio/high_max": 0.0017090555804315955, "clip_ratio/high_mean": 0.0017090555804315955, "clip_ratio/low_mean": 0.003607556634233333, "clip_ratio/low_min": 0.003607556634233333, "clip_ratio/region_mean": 0.005316612214664928, "completions/clipped_ratio": 0.2, "completions/max_length": 1783.0, "completions/max_terminated_length": 1542.3, "completions/mean_length": 1166.975, "completions/mean_terminated_length": 925.048583984375, "completions/min_length": 593.8, "completions/min_terminated_length": 593.8, "entropy": 0.23418510826304556, "epoch": 0.9037037037037037, "frac_reward_zero_std": 0.35, "grad_norm": 0.022599879652261734, "learning_rate": 4.863844439222215e-06, "loss": 0.0055918138474226, "num_tokens": 8046015.0, "reward": 0.6807381466031075, "reward_std": 0.4131760125979781, "rewards/correctness_reward/mean": 0.597500005364418, "rewards/correctness_reward/std": 0.38618521988391874, "rewards/format_reward/mean": 0.08000000193715096, "rewards/format_reward/std": 0.02811250314116478, "rewards/length_efficiency_reward/mean": 0.0032381250290200113, "rewards/length_efficiency_reward/std": 0.004869484226219356, "sampling/importance_sampling_ratio/max": 0.9369942888617515, "sampling/importance_sampling_ratio/mean": 0.16068057231605054, "sampling/importance_sampling_ratio/min": 0.00020102500302527916, "sampling/sampling_logp_difference/max": 2.823923945426941, "sampling/sampling_logp_difference/mean": 0.026291194930672647, "step": 610, "step_time": 13.503291972912848 }, { "clip_ratio/high_max": 0.0025066877642530017, "clip_ratio/high_mean": 0.0025066877642530017, "clip_ratio/low_mean": 0.0032100550160976127, "clip_ratio/low_min": 0.0032100550160976127, "clip_ratio/region_mean": 0.005716742780350614, "completions/clipped_ratio": 0.1625, "completions/max_length": 1679.3, "completions/max_terminated_length": 1417.7, "completions/mean_length": 1174.3625, "completions/mean_terminated_length": 1009.3500122070312, "completions/min_length": 654.6, "completions/min_terminated_length": 654.6, "entropy": 0.25175703670829536, "epoch": 0.9185185185185185, "frac_reward_zero_std": 0.3, "grad_norm": 0.4328603148460388, "learning_rate": 4.857120076513469e-06, "loss": 0.012734563648700714, "num_tokens": 8151520.0, "reward": 0.6541375005617738, "reward_std": 0.31360632060095667, "rewards/correctness_reward/mean": 0.5675000028684736, "rewards/correctness_reward/std": 0.28851412087678907, "rewards/format_reward/mean": 0.0837500024586916, "rewards/format_reward/std": 0.022220886126160622, "rewards/length_efficiency_reward/mean": 0.0028874999290565027, "rewards/length_efficiency_reward/std": 0.003978247789200395, "sampling/importance_sampling_ratio/max": 0.7307589180767536, "sampling/importance_sampling_ratio/mean": 0.14156947731971742, "sampling/importance_sampling_ratio/min": 0.00042785145255095356, "sampling/sampling_logp_difference/max": 2.7969128370285032, "sampling/sampling_logp_difference/mean": 0.02867441698908806, "step": 620, "step_time": 12.969812010508031 }, { "clip_ratio/high_max": 0.0018697180130402558, "clip_ratio/high_mean": 0.0018697180130402558, "clip_ratio/low_mean": 0.003414608075399883, "clip_ratio/low_min": 0.003414608075399883, "clip_ratio/region_mean": 0.005284326088440139, "completions/clipped_ratio": 0.3, "completions/max_length": 1724.6, "completions/max_terminated_length": 1568.8, "completions/mean_length": 1410.5, "completions/mean_terminated_length": 1260.71298828125, "completions/min_length": 999.2, "completions/min_terminated_length": 999.2, "entropy": 0.22688631480559707, "epoch": 0.9333333333333333, "frac_reward_zero_std": 0.35, "grad_norm": 0.23353974521160126, "learning_rate": 4.85023852027494e-06, "loss": -0.0385995477437973, "num_tokens": 8282376.0, "reward": 0.5005150139331818, "reward_std": 0.36423158645629883, "rewards/correctness_reward/mean": 0.4275000005960464, "rewards/correctness_reward/std": 0.3373343922197819, "rewards/format_reward/mean": 0.07000000160187483, "rewards/format_reward/std": 0.027227428182959555, "rewards/length_efficiency_reward/mean": 0.003014999913284555, "rewards/length_efficiency_reward/std": 0.0036945650179404767, "sampling/importance_sampling_ratio/max": 0.6271260648965835, "sampling/importance_sampling_ratio/mean": 0.1414671067148447, "sampling/importance_sampling_ratio/min": 0.0011586208878725302, "sampling/sampling_logp_difference/max": 1.9565863132476806, "sampling/sampling_logp_difference/mean": 0.023810026235878468, "step": 630, "step_time": 13.640375631675123 }, { "clip_ratio/high_max": 0.001827876886818558, "clip_ratio/high_mean": 0.001827876886818558, "clip_ratio/low_mean": 0.002332168389693834, "clip_ratio/low_min": 0.002332168389693834, "clip_ratio/region_mean": 0.004160045276512392, "completions/clipped_ratio": 0.25, "completions/max_length": 1804.2, "completions/max_terminated_length": 1430.8, "completions/mean_length": 1273.45, "completions/mean_terminated_length": 1019.7375122070313, "completions/min_length": 718.0, "completions/min_terminated_length": 718.0, "entropy": 0.23302650600671768, "epoch": 0.9481481481481482, "frac_reward_zero_std": 0.5, "grad_norm": 0.2499055117368698, "learning_rate": 4.843200229429415e-06, "loss": -0.016894568502902985, "num_tokens": 8395640.0, "reward": 0.5910850072279572, "reward_std": 0.4585941815748811, "rewards/correctness_reward/mean": 0.5125000011175871, "rewards/correctness_reward/std": 0.42455523237586024, "rewards/format_reward/mean": 0.07500000149011612, "rewards/format_reward/std": 0.030639102309942247, "rewards/length_efficiency_reward/mean": 0.0035849999519996347, "rewards/length_efficiency_reward/std": 0.0056838416261598464, "sampling/importance_sampling_ratio/max": 1.3166327148675918, "sampling/importance_sampling_ratio/mean": 0.2458759443834424, "sampling/importance_sampling_ratio/min": 0.0008547615380393836, "sampling/sampling_logp_difference/max": 2.542231047153473, "sampling/sampling_logp_difference/mean": 0.025940349511802196, "step": 640, "step_time": 13.717802858445793 }, { "clip_ratio/high_max": 0.0015855358156841247, "clip_ratio/high_mean": 0.0015855358156841247, "clip_ratio/low_mean": 0.0029956455546198413, "clip_ratio/low_min": 0.0029956455546198413, "clip_ratio/region_mean": 0.0045811813703039665, "completions/clipped_ratio": 0.0875, "completions/max_length": 1577.5, "completions/max_terminated_length": 1409.2, "completions/mean_length": 1078.05, "completions/mean_terminated_length": 977.6125, "completions/min_length": 643.2, "completions/min_terminated_length": 643.2, "entropy": 0.22282413011416793, "epoch": 0.9629629629629629, "frac_reward_zero_std": 0.45, "grad_norm": 0.002297706203535199, "learning_rate": 4.836005673352126e-06, "loss": -0.001988864317536354, "num_tokens": 8491872.0, "reward": 0.851255014538765, "reward_std": 0.2885954800818581, "rewards/correctness_reward/mean": 0.7575000017881394, "rewards/correctness_reward/std": 0.27576125264167783, "rewards/format_reward/mean": 0.0912500012665987, "rewards/format_reward/std": 0.01052071750164032, "rewards/length_efficiency_reward/mean": 0.0025049999298062176, "rewards/length_efficiency_reward/std": 0.0033295282744802536, "sampling/importance_sampling_ratio/max": 0.8961162328720093, "sampling/importance_sampling_ratio/mean": 0.19672916773706675, "sampling/importance_sampling_ratio/min": 0.00034397610434098167, "sampling/sampling_logp_difference/max": 2.458750867843628, "sampling/sampling_logp_difference/mean": 0.024802970141172408, "step": 650, "step_time": 12.262384535651654 }, { "clip_ratio/high_max": 0.0019325924658915028, "clip_ratio/high_mean": 0.0019325924658915028, "clip_ratio/low_mean": 0.0023062766704242676, "clip_ratio/low_min": 0.0023062766704242676, "clip_ratio/region_mean": 0.0042388691363157704, "completions/clipped_ratio": 0.2875, "completions/max_length": 1779.9, "completions/max_terminated_length": 1375.4, "completions/mean_length": 1299.3375, "completions/mean_terminated_length": 941.7732299804687, "completions/min_length": 818.3, "completions/min_terminated_length": 613.5, "entropy": 0.2523540424183011, "epoch": 0.9777777777777777, "frac_reward_zero_std": 0.55, "grad_norm": 0.1607706993818283, "learning_rate": 4.828655331839446e-06, "loss": -0.004961637780070305, "num_tokens": 8607711.0, "reward": 0.49160062447190284, "reward_std": 0.3030946939252317, "rewards/correctness_reward/mean": 0.41750000417232513, "rewards/correctness_reward/std": 0.2772039145231247, "rewards/format_reward/mean": 0.07125000245869159, "rewards/format_reward/std": 0.02664227820932865, "rewards/length_efficiency_reward/mean": 0.002850625035353005, "rewards/length_efficiency_reward/std": 0.004121143440715969, "sampling/importance_sampling_ratio/max": 0.8931512862443924, "sampling/importance_sampling_ratio/mean": 0.205360247194767, "sampling/importance_sampling_ratio/min": 0.0011615223028911715, "sampling/sampling_logp_difference/max": 2.2872172832489013, "sampling/sampling_logp_difference/mean": 0.02588100340217352, "step": 660, "step_time": 13.565898792818189 }, { "clip_ratio/high_max": 0.0016453732387162745, "clip_ratio/high_mean": 0.0016453732387162745, "clip_ratio/low_mean": 0.003305859403917566, "clip_ratio/low_min": 0.003305859403917566, "clip_ratio/region_mean": 0.00495123264263384, "completions/clipped_ratio": 0.3875, "completions/max_length": 1786.0, "completions/max_terminated_length": 1518.9, "completions/mean_length": 1459.225, "completions/mean_terminated_length": 1248.5450073242187, "completions/min_length": 1037.4, "completions/min_terminated_length": 1037.4, "entropy": 0.26288966052234175, "epoch": 0.9925925925925926, "frac_reward_zero_std": 0.45, "grad_norm": 0.0, "learning_rate": 4.821149695076895e-06, "loss": -0.008715462684631348, "num_tokens": 8736685.0, "reward": 0.33507501063868406, "reward_std": 0.5103490695357322, "rewards/correctness_reward/mean": 0.27250000163912774, "rewards/correctness_reward/std": 0.4766154192388058, "rewards/format_reward/mean": 0.06125000072643161, "rewards/format_reward/std": 0.03455129489302635, "rewards/length_efficiency_reward/mean": 0.0013250000192783773, "rewards/length_efficiency_reward/std": 0.0022042951779440044, "sampling/importance_sampling_ratio/max": 0.8314185961149633, "sampling/importance_sampling_ratio/mean": 0.19000481935217978, "sampling/importance_sampling_ratio/min": 0.0004321204723964911, "sampling/sampling_logp_difference/max": 2.15596364736557, "sampling/sampling_logp_difference/mean": 0.025060435570776463, "step": 670, "step_time": 13.750607237312943 }, { "epoch": 1.0, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.185, "eval_completions/max_length": 1733.8266666666666, "eval_completions/max_terminated_length": 1517.6, "eval_completions/mean_length": 1269.045, "eval_completions/mean_terminated_length": 1103.2449308268228, "eval_completions/min_length": 803.7866666666666, "eval_completions/min_terminated_length": 776.48, "eval_entropy": 0.2435171588261922, "eval_frac_reward_zero_std": 0.5, "eval_loss": -0.001249276683665812, "eval_num_tokens": 8790836.0, "eval_reward": 0.6283912657573819, "eval_reward_std": 0.3598713335379337, "eval_rewards/correctness_reward/mean": 0.5456666683157285, "eval_rewards/correctness_reward/std": 0.33928621649742124, "eval_rewards/format_reward/mean": 0.08116666855911414, "eval_rewards/format_reward/std": 0.022064068963130314, "eval_rewards/length_efficiency_reward/mean": 0.00155791665674163, "eval_rewards/length_efficiency_reward/std": 0.0018331278715534912, "eval_runtime": 765.4453, "eval_samples_per_second": 0.196, "eval_sampling/importance_sampling_ratio/max": 1.1814488777518273, "eval_sampling/importance_sampling_ratio/mean": 0.2527447059750557, "eval_sampling/importance_sampling_ratio/min": 0.0020784807143855726, "eval_sampling/sampling_logp_difference/max": 2.0617604716618856, "eval_sampling/sampling_logp_difference/mean": 0.023671377363304297, "eval_steps_per_second": 0.025, "step": 675 }, { "clip_ratio/high_max": 0.0015264986956026404, "clip_ratio/high_mean": 0.0015264986956026404, "clip_ratio/low_mean": 0.002290237316628918, "clip_ratio/low_min": 0.002290237316628918, "clip_ratio/region_mean": 0.0038167360122315585, "completions/clipped_ratio": 0.125, "completions/max_length": 1704.6, "completions/max_terminated_length": 1468.5, "completions/mean_length": 1244.8375, "completions/mean_terminated_length": 1115.9125122070313, "completions/min_length": 825.3, "completions/min_terminated_length": 825.3, "entropy": 0.2637187363579869, "epoch": 1.0074074074074073, "frac_reward_zero_std": 0.55, "grad_norm": 0.03154218941926956, "learning_rate": 4.813489263606445e-06, "loss": -0.030103567242622375, "num_tokens": 8848568.0, "reward": 0.6638918917626142, "reward_std": 0.39938992911484095, "rewards/correctness_reward/mean": 0.5749999992549419, "rewards/correctness_reward/std": 0.38337581157684325, "rewards/format_reward/mean": 0.08750000223517418, "rewards/format_reward/std": 0.017761518806219102, "rewards/length_efficiency_reward/mean": 0.0013918749173171818, "rewards/length_efficiency_reward/std": 0.002168681425973773, "sampling/importance_sampling_ratio/max": 1.1031194746494293, "sampling/importance_sampling_ratio/mean": 0.1888826590962708, "sampling/importance_sampling_ratio/min": 0.0008756305018323473, "sampling/sampling_logp_difference/max": 1.6112547159194945, "sampling/sampling_logp_difference/mean": 0.025913743115961552, "step": 680, "step_time": 12.636950113810599 }, { "clip_ratio/high_max": 0.0014488026936305687, "clip_ratio/high_mean": 0.0014488026936305687, "clip_ratio/low_mean": 0.002561548206722364, "clip_ratio/low_min": 0.002561548206722364, "clip_ratio/region_mean": 0.004010350900352932, "completions/clipped_ratio": 0.2125, "completions/max_length": 1751.8, "completions/max_terminated_length": 1543.6, "completions/mean_length": 1262.9125, "completions/mean_terminated_length": 1070.9070373535155, "completions/min_length": 760.2, "completions/min_terminated_length": 760.2, "entropy": 0.24690341260284185, "epoch": 1.0222222222222221, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 4.805674548293148e-06, "loss": 0.029953080415725707, "num_tokens": 8962437.0, "reward": 0.7004637735895812, "reward_std": 0.3366726465988904, "rewards/correctness_reward/mean": 0.6200000002980233, "rewards/correctness_reward/std": 0.3087812066078186, "rewards/format_reward/mean": 0.07875000163912774, "rewards/format_reward/std": 0.02920606955885887, "rewards/length_efficiency_reward/mean": 0.00171374999335967, "rewards/length_efficiency_reward/std": 0.002413355279713869, "sampling/importance_sampling_ratio/max": 1.255669939517975, "sampling/importance_sampling_ratio/mean": 0.22989916801452637, "sampling/importance_sampling_ratio/min": 0.000299283553067653, "sampling/sampling_logp_difference/max": 2.49380224943161, "sampling/sampling_logp_difference/mean": 0.02519239205867052, "step": 690, "step_time": 12.983375156950206 }, { "clip_ratio/high_max": 0.0008610342367319391, "clip_ratio/high_mean": 0.0008610342367319391, "clip_ratio/low_mean": 0.002064660296309739, "clip_ratio/low_min": 0.002064660296309739, "clip_ratio/region_mean": 0.0029256945330416785, "completions/clipped_ratio": 0.25, "completions/max_length": 1858.3, "completions/max_terminated_length": 1557.4, "completions/mean_length": 1368.575, "completions/mean_terminated_length": 1134.3483520507812, "completions/min_length": 855.5, "completions/min_terminated_length": 855.5, "entropy": 0.24074180321767927, "epoch": 1.037037037037037, "frac_reward_zero_std": 0.6, "grad_norm": 0.12793712317943573, "learning_rate": 4.797706070291054e-06, "loss": 0.017122888565063478, "num_tokens": 9082463.0, "reward": 0.5222581423819065, "reward_std": 0.5017222464084625, "rewards/correctness_reward/mean": 0.4474999994039536, "rewards/correctness_reward/std": 0.47364020347595215, "rewards/format_reward/mean": 0.07375000156462193, "rewards/format_reward/std": 0.030129902809858323, "rewards/length_efficiency_reward/mean": 0.001008124987129122, "rewards/length_efficiency_reward/std": 0.001519506238400936, "sampling/importance_sampling_ratio/max": 0.8391612686216832, "sampling/importance_sampling_ratio/mean": 0.1979271860793233, "sampling/importance_sampling_ratio/min": 0.0009850131486018655, "sampling/sampling_logp_difference/max": 2.265767526626587, "sampling/sampling_logp_difference/mean": 0.02310623601078987, "step": 700, "step_time": 13.400787764601409 }, { "clip_ratio/high_max": 0.0011302793835056945, "clip_ratio/high_mean": 0.0011302793835056945, "clip_ratio/low_mean": 0.0032032409246312453, "clip_ratio/low_min": 0.0032032409246312453, "clip_ratio/region_mean": 0.00433352030813694, "completions/clipped_ratio": 0.3125, "completions/max_length": 1918.3, "completions/max_terminated_length": 1428.1, "completions/mean_length": 1394.725, "completions/mean_terminated_length": 967.4058349609375, "completions/min_length": 883.9, "completions/min_terminated_length": 679.1, "entropy": 0.2528625194914639, "epoch": 1.0518518518518518, "frac_reward_zero_std": 0.45, "grad_norm": 0.0038879932835698128, "learning_rate": 4.789584361008471e-06, "loss": 0.0032389026135206224, "num_tokens": 9205477.0, "reward": 0.4450093932449818, "reward_std": 0.384121474949643, "rewards/correctness_reward/mean": 0.3749999970197678, "rewards/correctness_reward/std": 0.35534276366233825, "rewards/format_reward/mean": 0.06875000149011612, "rewards/format_reward/std": 0.029753244295716286, "rewards/length_efficiency_reward/mean": 0.0012593750143423676, "rewards/length_efficiency_reward/std": 0.0018799818819388748, "sampling/importance_sampling_ratio/max": 1.104397924989462, "sampling/importance_sampling_ratio/mean": 0.2476374063640833, "sampling/importance_sampling_ratio/min": 0.0006203525531625212, "sampling/sampling_logp_difference/max": 2.7196027994155885, "sampling/sampling_logp_difference/mean": 0.023784328438341617, "step": 710, "step_time": 13.802170952875167 }, { "clip_ratio/high_max": 0.0014338629902340473, "clip_ratio/high_mean": 0.0014338629902340473, "clip_ratio/low_mean": 0.002420046311453916, "clip_ratio/low_min": 0.002420046311453916, "clip_ratio/region_mean": 0.0038539093016879632, "completions/clipped_ratio": 0.275, "completions/max_length": 1992.5, "completions/max_terminated_length": 1512.4, "completions/mean_length": 1376.8875, "completions/mean_terminated_length": 1097.4361999511718, "completions/min_length": 778.8, "completions/min_terminated_length": 778.8, "entropy": 0.2641914011910558, "epoch": 1.0666666666666667, "frac_reward_zero_std": 0.55, "grad_norm": 0.0, "learning_rate": 4.781309962072514e-06, "loss": -0.01075884997844696, "num_tokens": 9327036.0, "reward": 0.5686843901872635, "reward_std": 0.6213426947593689, "rewards/correctness_reward/mean": 0.4950000047683716, "rewards/correctness_reward/std": 0.5874309480190277, "rewards/format_reward/mean": 0.07250000201165677, "rewards/format_reward/std": 0.038086828961968425, "rewards/length_efficiency_reward/mean": 0.001184375016600825, "rewards/length_efficiency_reward/std": 0.0017444209690438584, "sampling/importance_sampling_ratio/max": 1.1087198555469513, "sampling/importance_sampling_ratio/mean": 0.17728611323982477, "sampling/importance_sampling_ratio/min": 0.0002821718244376825, "sampling/sampling_logp_difference/max": 2.22471227645874, "sampling/sampling_logp_difference/mean": 0.024973604641854764, "step": 720, "step_time": 14.209118811320513 }, { "clip_ratio/high_max": 0.001068336478783749, "clip_ratio/high_mean": 0.001068336478783749, "clip_ratio/low_mean": 0.0030173318518791348, "clip_ratio/low_min": 0.0030173318518791348, "clip_ratio/region_mean": 0.004085668330662884, "completions/clipped_ratio": 0.125, "completions/max_length": 1692.2, "completions/max_terminated_length": 1457.4, "completions/mean_length": 1180.475, "completions/mean_terminated_length": 1044.6882202148438, "completions/min_length": 748.7, "completions/min_terminated_length": 748.7, "entropy": 0.24568698797374963, "epoch": 1.0814814814814815, "frac_reward_zero_std": 0.45, "grad_norm": 0.10271739959716797, "learning_rate": 4.772883425292991e-06, "loss": -0.012519745528697968, "num_tokens": 9431990.0, "reward": 0.6639443868771195, "reward_std": 0.4453204446937889, "rewards/correctness_reward/mean": 0.5750000040978194, "rewards/correctness_reward/std": 0.42540021166205405, "rewards/format_reward/mean": 0.08750000186264514, "rewards/format_reward/std": 0.01868535205721855, "rewards/length_efficiency_reward/mean": 0.0014443749852944166, "rewards/length_efficiency_reward/std": 0.003260852780658752, "sampling/importance_sampling_ratio/max": 0.9046363361179829, "sampling/importance_sampling_ratio/mean": 0.1814103739336133, "sampling/importance_sampling_ratio/min": 0.00039193501870613543, "sampling/sampling_logp_difference/max": 2.361107861995697, "sampling/sampling_logp_difference/mean": 0.02569099385291338, "step": 730, "step_time": 12.404605556558817 }, { "clip_ratio/high_max": 0.0015705125042586587, "clip_ratio/high_mean": 0.0015705125042586587, "clip_ratio/low_mean": 0.0027972771466011183, "clip_ratio/low_min": 0.0027972771466011183, "clip_ratio/region_mean": 0.004367789650859777, "completions/clipped_ratio": 0.2, "completions/max_length": 1775.0, "completions/max_terminated_length": 1561.1, "completions/mean_length": 1284.275, "completions/mean_terminated_length": 1117.9416748046874, "completions/min_length": 806.6, "completions/min_terminated_length": 806.6, "entropy": 0.24806692264974117, "epoch": 1.0962962962962963, "frac_reward_zero_std": 0.45, "grad_norm": 0.017257433384656906, "learning_rate": 4.764305312625601e-06, "loss": 0.02394607663154602, "num_tokens": 9547520.0, "reward": 0.6177056366577744, "reward_std": 0.38184614330530164, "rewards/correctness_reward/mean": 0.5350000001490116, "rewards/correctness_reward/std": 0.35785320624709127, "rewards/format_reward/mean": 0.08000000156462192, "rewards/format_reward/std": 0.025293876975774766, "rewards/length_efficiency_reward/mean": 0.00270562507212162, "rewards/length_efficiency_reward/std": 0.003223613905720413, "sampling/importance_sampling_ratio/max": 1.1264439798891543, "sampling/importance_sampling_ratio/mean": 0.22419266998767853, "sampling/importance_sampling_ratio/min": 0.0023799622109436313, "sampling/sampling_logp_difference/max": 1.9480568289756774, "sampling/sampling_logp_difference/mean": 0.022884450294077395, "step": 740, "step_time": 13.017312775924802 }, { "clip_ratio/high_max": 0.001975291603594087, "clip_ratio/high_mean": 0.001975291603594087, "clip_ratio/low_mean": 0.0031144936656346545, "clip_ratio/low_min": 0.0031144936656346545, "clip_ratio/region_mean": 0.005089785269228741, "completions/clipped_ratio": 0.225, "completions/max_length": 1653.1, "completions/max_terminated_length": 1432.8, "completions/mean_length": 1234.6375, "completions/mean_terminated_length": 1063.18037109375, "completions/min_length": 787.9, "completions/min_terminated_length": 787.9, "entropy": 0.28750082328915594, "epoch": 1.1111111111111112, "frac_reward_zero_std": 0.4, "grad_norm": 0.05655239522457123, "learning_rate": 4.755576196134458e-06, "loss": 0.01219373643398285, "num_tokens": 9657739.0, "reward": 0.7080575078725815, "reward_std": 0.38661226438416635, "rewards/correctness_reward/mean": 0.630000001937151, "rewards/correctness_reward/std": 0.36465256810188296, "rewards/format_reward/mean": 0.07750000189989806, "rewards/format_reward/std": 0.023314452543854714, "rewards/length_efficiency_reward/mean": 0.0005575000068347435, "rewards/length_efficiency_reward/std": 0.0011469640376162715, "sampling/importance_sampling_ratio/max": 0.8337138421833515, "sampling/importance_sampling_ratio/mean": 0.17275433503091336, "sampling/importance_sampling_ratio/min": 0.0014555117122654337, "sampling/sampling_logp_difference/max": 1.8828314542770386, "sampling/sampling_logp_difference/mean": 0.026723692007362843, "step": 750, "step_time": 12.372394063137472 }, { "clip_ratio/high_max": 0.003251469787210226, "clip_ratio/high_mean": 0.003251469787210226, "clip_ratio/low_mean": 0.0024529346730560063, "clip_ratio/low_min": 0.0024529346730560063, "clip_ratio/region_mean": 0.005704404460266233, "completions/clipped_ratio": 0.3625, "completions/max_length": 2032.1, "completions/max_terminated_length": 1778.6, "completions/mean_length": 1588.975, "completions/mean_terminated_length": 1353.750732421875, "completions/min_length": 955.4, "completions/min_terminated_length": 955.4, "entropy": 0.2565307702869177, "epoch": 1.125925925925926, "frac_reward_zero_std": 0.25, "grad_norm": 1.0087616443634033, "learning_rate": 4.746696657953942e-06, "loss": -0.030097579956054686, "num_tokens": 9798649.0, "reward": 0.45603815242648127, "reward_std": 0.5495598331093788, "rewards/correctness_reward/mean": 0.39000000022351744, "rewards/correctness_reward/std": 0.5091322720050812, "rewards/format_reward/mean": 0.06375000178813935, "rewards/format_reward/std": 0.04452562071383, "rewards/length_efficiency_reward/mean": 0.0022881250508362426, "rewards/length_efficiency_reward/std": 0.004452742531429976, "sampling/importance_sampling_ratio/max": 0.991300418972969, "sampling/importance_sampling_ratio/mean": 0.2592890053987503, "sampling/importance_sampling_ratio/min": 0.0004760694213928218, "sampling/sampling_logp_difference/max": 1.4907065153121948, "sampling/sampling_logp_difference/mean": 0.022413095645606517, "step": 760, "step_time": 14.737608800828458 }, { "clip_ratio/high_max": 0.0016238999989582226, "clip_ratio/high_mean": 0.0016238999989582226, "clip_ratio/low_mean": 0.0032967827733955345, "clip_ratio/low_min": 0.0032967827733955345, "clip_ratio/region_mean": 0.004920682772353757, "completions/clipped_ratio": 0.3125, "completions/max_length": 1934.3, "completions/max_terminated_length": 1476.1, "completions/mean_length": 1385.2125, "completions/mean_terminated_length": 1078.4208374023438, "completions/min_length": 774.7, "completions/min_terminated_length": 774.7, "entropy": 0.23961996594443918, "epoch": 1.1407407407407408, "frac_reward_zero_std": 0.35, "grad_norm": 0.014492428861558437, "learning_rate": 4.737667290249874e-06, "loss": -0.02881160080432892, "num_tokens": 9921566.0, "reward": 0.48685251716524364, "reward_std": 0.5765734702348709, "rewards/correctness_reward/mean": 0.41500000171363355, "rewards/correctness_reward/std": 0.5398185215890408, "rewards/format_reward/mean": 0.07000000141561032, "rewards/format_reward/std": 0.03564486131072044, "rewards/length_efficiency_reward/mean": 0.0018524999897636008, "rewards/length_efficiency_reward/std": 0.002913024956069421, "sampling/importance_sampling_ratio/max": 1.2050759091973304, "sampling/importance_sampling_ratio/mean": 0.2479958117008209, "sampling/importance_sampling_ratio/min": 0.0010086745303851785, "sampling/sampling_logp_difference/max": 1.6015749812126159, "sampling/sampling_logp_difference/mean": 0.02270013764500618, "step": 770, "step_time": 13.878034065384417 }, { "clip_ratio/high_max": 0.001718764120596461, "clip_ratio/high_mean": 0.001718764120596461, "clip_ratio/low_mean": 0.003720205355784856, "clip_ratio/low_min": 0.003720205355784856, "clip_ratio/region_mean": 0.0054389694763813164, "completions/clipped_ratio": 0.0625, "completions/max_length": 1749.7, "completions/max_terminated_length": 1718.0, "completions/mean_length": 1236.925, "completions/mean_terminated_length": 1205.5517944335938, "completions/min_length": 779.2, "completions/min_terminated_length": 779.2, "entropy": 0.24089245647192, "epoch": 1.1555555555555554, "frac_reward_zero_std": 0.4, "grad_norm": 0.00441139517351985, "learning_rate": 4.7284886951800325e-06, "loss": -0.059500879049301146, "num_tokens": 10031356.0, "reward": 0.8219750195741653, "reward_std": 0.289375511999242, "rewards/correctness_reward/mean": 0.725, "rewards/correctness_reward/std": 0.27874754667282103, "rewards/format_reward/mean": 0.09375000186264515, "rewards/format_reward/std": 0.008880759403109551, "rewards/length_efficiency_reward/mean": 0.0032250000804197043, "rewards/length_efficiency_reward/std": 0.004417234042193741, "sampling/importance_sampling_ratio/max": 1.0817362666130066, "sampling/importance_sampling_ratio/mean": 0.2113557480275631, "sampling/importance_sampling_ratio/min": 0.0019697159919360276, "sampling/sampling_logp_difference/max": 2.034777820110321, "sampling/sampling_logp_difference/mean": 0.025384920462965966, "step": 780, "step_time": 12.717394668515771 }, { "clip_ratio/high_max": 0.0015456722729140892, "clip_ratio/high_mean": 0.0015456722729140892, "clip_ratio/low_mean": 0.0031376114406157286, "clip_ratio/low_min": 0.0031376114406157286, "clip_ratio/region_mean": 0.004683283713529818, "completions/clipped_ratio": 0.175, "completions/max_length": 1489.2, "completions/max_terminated_length": 1238.3, "completions/mean_length": 1083.0125, "completions/mean_terminated_length": 878.2625, "completions/min_length": 674.4, "completions/min_terminated_length": 674.4, "entropy": 0.2376161266118288, "epoch": 1.1703703703703703, "frac_reward_zero_std": 0.45, "grad_norm": 0.03544621542096138, "learning_rate": 4.7191614848539855e-06, "loss": -0.03584673702716827, "num_tokens": 10129601.0, "reward": 0.6625131540000438, "reward_std": 0.28609858294948937, "rewards/correctness_reward/mean": 0.5774999994784593, "rewards/correctness_reward/std": 0.27121175453066826, "rewards/format_reward/mean": 0.08250000113621354, "rewards/format_reward/std": 0.01405625157058239, "rewards/length_efficiency_reward/mean": 0.002513125035329722, "rewards/length_efficiency_reward/std": 0.0035356479114852847, "sampling/importance_sampling_ratio/max": 1.1885975174605847, "sampling/importance_sampling_ratio/mean": 0.19246141561307012, "sampling/importance_sampling_ratio/min": 0.0009629759841118357, "sampling/sampling_logp_difference/max": 2.1346989393234255, "sampling/sampling_logp_difference/mean": 0.027653400413692, "step": 790, "step_time": 11.328566823061555 }, { "clip_ratio/high_max": 0.0019758769951295106, "clip_ratio/high_mean": 0.0019758769951295106, "clip_ratio/low_mean": 0.0034003025037236513, "clip_ratio/low_min": 0.0034003025037236513, "clip_ratio/region_mean": 0.005376179498853162, "completions/clipped_ratio": 0.1625, "completions/max_length": 1980.5, "completions/max_terminated_length": 1859.1, "completions/mean_length": 1389.0, "completions/mean_terminated_length": 1255.3607421875, "completions/min_length": 896.6, "completions/min_terminated_length": 896.6, "entropy": 0.2505159616470337, "epoch": 1.1851851851851851, "frac_reward_zero_std": 0.4, "grad_norm": 0.02673676237463951, "learning_rate": 4.709686281292277e-06, "loss": 0.011465317010879517, "num_tokens": 10253481.0, "reward": 0.6188206508755684, "reward_std": 0.4785047501325607, "rewards/correctness_reward/mean": 0.5324999988079071, "rewards/correctness_reward/std": 0.4585972994565964, "rewards/format_reward/mean": 0.08500000201165676, "rewards/format_reward/std": 0.023314452543854714, "rewards/length_efficiency_reward/mean": 0.0013206249568611383, "rewards/length_efficiency_reward/std": 0.0022915847599506377, "sampling/importance_sampling_ratio/max": 0.5059421837329865, "sampling/importance_sampling_ratio/mean": 0.13455926030874252, "sampling/importance_sampling_ratio/min": 0.0005626860947359092, "sampling/sampling_logp_difference/max": 2.331648921966553, "sampling/sampling_logp_difference/mean": 0.026670989394187928, "step": 800, "step_time": 14.177801271993667 }, { "clip_ratio/high_max": 0.002236992739199195, "clip_ratio/high_mean": 0.002236992739199195, "clip_ratio/low_mean": 0.002401618522708304, "clip_ratio/low_min": 0.002401618522708304, "clip_ratio/region_mean": 0.004638611261907499, "completions/clipped_ratio": 0.175, "completions/max_length": 1685.5, "completions/max_terminated_length": 1284.2, "completions/mean_length": 1225.4875, "completions/mean_terminated_length": 940.3678588867188, "completions/min_length": 883.4, "completions/min_terminated_length": 678.6, "entropy": 0.21597187900915743, "epoch": 1.2, "frac_reward_zero_std": 0.45, "grad_norm": 0.6308737993240356, "learning_rate": 4.700063716384947e-06, "loss": -0.035885027050971983, "num_tokens": 10362732.0, "reward": 0.6506143853068351, "reward_std": 0.2903867067187093, "rewards/correctness_reward/mean": 0.5650000026449561, "rewards/correctness_reward/std": 0.2756091557443142, "rewards/format_reward/mean": 0.08250000216066837, "rewards/format_reward/std": 0.01241629347205162, "rewards/length_efficiency_reward/mean": 0.00311437501659384, "rewards/length_efficiency_reward/std": 0.003996050718706101, "sampling/importance_sampling_ratio/max": 1.0314480438828468, "sampling/importance_sampling_ratio/mean": 0.24575295709073544, "sampling/importance_sampling_ratio/min": 0.006727096897029128, "sampling/sampling_logp_difference/max": 2.0642957210540773, "sampling/sampling_logp_difference/mean": 0.025215629301965237, "step": 810, "step_time": 12.447143352311105 }, { "clip_ratio/high_max": 0.0017469869198976085, "clip_ratio/high_mean": 0.0017469869198976085, "clip_ratio/low_mean": 0.002606634743278846, "clip_ratio/low_min": 0.002606634743278846, "clip_ratio/region_mean": 0.004353621663176455, "completions/clipped_ratio": 0.3625, "completions/max_length": 1796.9, "completions/max_terminated_length": 1469.1, "completions/mean_length": 1439.1625, "completions/mean_terminated_length": 1177.525, "completions/min_length": 965.7, "completions/min_terminated_length": 965.7, "entropy": 0.23503528591245412, "epoch": 1.2148148148148148, "frac_reward_zero_std": 0.5, "grad_norm": 0.5622824430465698, "learning_rate": 4.690294431849385e-06, "loss": 0.03817010223865509, "num_tokens": 10492117.0, "reward": 0.4677656374871731, "reward_std": 0.3717491101939231, "rewards/correctness_reward/mean": 0.40250000171363354, "rewards/correctness_reward/std": 0.3381554685533047, "rewards/format_reward/mean": 0.06375000122934579, "rewards/format_reward/std": 0.035814594477415085, "rewards/length_efficiency_reward/mean": 0.0015156249981373548, "rewards/length_efficiency_reward/std": 0.0025761883240193128, "sampling/importance_sampling_ratio/max": 0.7831450559198856, "sampling/importance_sampling_ratio/mean": 0.1765402304008603, "sampling/importance_sampling_ratio/min": 0.00040981352794915437, "sampling/sampling_logp_difference/max": 2.239521396160126, "sampling/sampling_logp_difference/mean": 0.02423766180872917, "step": 820, "step_time": 13.41466003730893 }, { "clip_ratio/high_max": 0.002006589621305466, "clip_ratio/high_mean": 0.002006589621305466, "clip_ratio/low_mean": 0.002845296775922179, "clip_ratio/low_min": 0.002845296775922179, "clip_ratio/region_mean": 0.004851886397227645, "completions/clipped_ratio": 0.2, "completions/max_length": 1577.0, "completions/max_terminated_length": 1309.6, "completions/mean_length": 1134.5125, "completions/mean_terminated_length": 980.5035766601562, "completions/min_length": 763.0, "completions/min_terminated_length": 763.0, "entropy": 0.2254137484356761, "epoch": 1.2296296296296296, "frac_reward_zero_std": 0.5, "grad_norm": 0.03234013915061951, "learning_rate": 4.680379079187536e-06, "loss": 0.03971203863620758, "num_tokens": 10595374.0, "reward": 0.542868772894144, "reward_std": 0.27034417442046105, "rewards/correctness_reward/mean": 0.4599999999627471, "rewards/correctness_reward/std": 0.2493320271372795, "rewards/format_reward/mean": 0.08000000165775419, "rewards/format_reward/std": 0.017761518806219102, "rewards/length_efficiency_reward/mean": 0.0028687500045634804, "rewards/length_efficiency_reward/std": 0.003801924083381891, "sampling/importance_sampling_ratio/max": 0.8236501246690751, "sampling/importance_sampling_ratio/mean": 0.16517218947410583, "sampling/importance_sampling_ratio/min": 0.0006199702467711177, "sampling/sampling_logp_difference/max": 1.7686675548553468, "sampling/sampling_logp_difference/mean": 0.02469415236264467, "step": 830, "step_time": 11.879510245937855 }, { "clip_ratio/high_max": 0.0021400121637270785, "clip_ratio/high_mean": 0.0021400121637270785, "clip_ratio/low_mean": 0.003409858557279222, "clip_ratio/low_min": 0.003409858557279222, "clip_ratio/region_mean": 0.0055498707210063, "completions/clipped_ratio": 0.1, "completions/max_length": 1554.9, "completions/max_terminated_length": 1485.3, "completions/mean_length": 1111.4375, "completions/mean_terminated_length": 1019.2816711425781, "completions/min_length": 644.5, "completions/min_terminated_length": 644.5, "entropy": 0.222479893732816, "epoch": 1.2444444444444445, "frac_reward_zero_std": 0.35, "grad_norm": 0.2251596450805664, "learning_rate": 4.6703183196424585e-06, "loss": -0.040441715717315675, "num_tokens": 10695373.0, "reward": 0.8612306505441666, "reward_std": 0.3049085272476077, "rewards/correctness_reward/mean": 0.7675000011920929, "rewards/correctness_reward/std": 0.2863773047924042, "rewards/format_reward/mean": 0.0900000013411045, "rewards/format_reward/std": 0.01498008482158184, "rewards/length_efficiency_reward/mean": 0.003730624984018505, "rewards/length_efficiency_reward/std": 0.004902051342651248, "sampling/importance_sampling_ratio/max": 0.7117771074175835, "sampling/importance_sampling_ratio/mean": 0.15312089063227177, "sampling/importance_sampling_ratio/min": 0.0022026808017471923, "sampling/sampling_logp_difference/max": 2.0287702798843386, "sampling/sampling_logp_difference/mean": 0.02509091291576624, "step": 840, "step_time": 11.678511502873153 }, { "clip_ratio/high_max": 0.001811630793963559, "clip_ratio/high_mean": 0.001811630793963559, "clip_ratio/low_mean": 0.003296306449919939, "clip_ratio/low_min": 0.003296306449919939, "clip_ratio/region_mean": 0.005107937243883498, "completions/clipped_ratio": 0.2625, "completions/max_length": 1897.6, "completions/max_terminated_length": 1387.5, "completions/mean_length": 1256.975, "completions/mean_terminated_length": 978.7518005371094, "completions/min_length": 633.0, "completions/min_terminated_length": 633.0, "entropy": 0.21787337306886911, "epoch": 1.2592592592592593, "frac_reward_zero_std": 0.4, "grad_norm": 0.013907878659665585, "learning_rate": 4.660112824154219e-06, "loss": 0.06324480175971985, "num_tokens": 10807399.0, "reward": 0.48669125959277154, "reward_std": 0.49485177649185064, "rewards/correctness_reward/mean": 0.41000000238418577, "rewards/correctness_reward/std": 0.46231070458889006, "rewards/format_reward/mean": 0.07375000193715095, "rewards/format_reward/std": 0.034174636378884314, "rewards/length_efficiency_reward/mean": 0.002941250003641471, "rewards/length_efficiency_reward/std": 0.004584773804526776, "sampling/importance_sampling_ratio/max": 0.9662797145545483, "sampling/importance_sampling_ratio/mean": 0.22485253009945155, "sampling/importance_sampling_ratio/min": 0.0011675120065774535, "sampling/sampling_logp_difference/max": 2.225196433067322, "sampling/sampling_logp_difference/mean": 0.023400415666401386, "step": 850, "step_time": 13.600520702358335 }, { "clip_ratio/high_max": 0.0023460826152586376, "clip_ratio/high_mean": 0.0023460826152586376, "clip_ratio/low_mean": 0.003597369749331847, "clip_ratio/low_min": 0.003597369749331847, "clip_ratio/region_mean": 0.005943452364590485, "completions/clipped_ratio": 0.3, "completions/max_length": 1680.8, "completions/max_terminated_length": 1357.0, "completions/mean_length": 1403.2125, "completions/mean_terminated_length": 1096.6111938476563, "completions/min_length": 1000.4, "completions/min_terminated_length": 795.6, "entropy": 0.2342445082962513, "epoch": 1.2740740740740741, "frac_reward_zero_std": 0.3, "grad_norm": 0.10757100582122803, "learning_rate": 4.649763273315156e-06, "loss": 0.015593291819095611, "num_tokens": 10933396.0, "reward": 0.5254250213503837, "reward_std": 0.3726277873851359, "rewards/correctness_reward/mean": 0.45250000040978194, "rewards/correctness_reward/std": 0.347893538326025, "rewards/format_reward/mean": 0.07000000160187483, "rewards/format_reward/std": 0.023314452543854714, "rewards/length_efficiency_reward/mean": 0.0029250000516185537, "rewards/length_efficiency_reward/std": 0.0037525434978306293, "sampling/importance_sampling_ratio/max": 1.0168635100126266, "sampling/importance_sampling_ratio/mean": 0.2094801241531968, "sampling/importance_sampling_ratio/min": 0.0007545235486759339, "sampling/sampling_logp_difference/max": 2.022670292854309, "sampling/sampling_logp_difference/mean": 0.0258089454844594, "step": 860, "step_time": 12.865429060626775 }, { "clip_ratio/high_max": 0.0016182168488739989, "clip_ratio/high_mean": 0.0016182168488739989, "clip_ratio/low_mean": 0.0038863588299136607, "clip_ratio/low_min": 0.0038863588299136607, "clip_ratio/region_mean": 0.00550457567878766, "completions/clipped_ratio": 0.2, "completions/max_length": 1832.9, "completions/max_terminated_length": 1511.3, "completions/mean_length": 1307.05, "completions/mean_terminated_length": 1105.3154907226562, "completions/min_length": 764.1, "completions/min_terminated_length": 764.1, "entropy": 0.2404268778860569, "epoch": 1.2888888888888888, "frac_reward_zero_std": 0.4, "grad_norm": 0.0887327790260315, "learning_rate": 4.639270357324484e-06, "loss": 0.0071077533066272736, "num_tokens": 11048836.0, "reward": 0.47899812534451486, "reward_std": 0.39295919227879494, "rewards/correctness_reward/mean": 0.39750000312924383, "rewards/correctness_reward/std": 0.37081067860126493, "rewards/format_reward/mean": 0.08000000193715096, "rewards/format_reward/std": 0.028659677878022195, "rewards/length_efficiency_reward/mean": 0.0014981250045821072, "rewards/length_efficiency_reward/std": 0.002778159989975393, "sampling/importance_sampling_ratio/max": 0.6855245185084641, "sampling/importance_sampling_ratio/mean": 0.14471053155139088, "sampling/importance_sampling_ratio/min": 0.0012290458633287926, "sampling/sampling_logp_difference/max": 1.9967660307884216, "sampling/sampling_logp_difference/mean": 0.02623310685157776, "step": 870, "step_time": 13.323789340164513 }, { "clip_ratio/high_max": 0.003150692762574181, "clip_ratio/high_mean": 0.003150692762574181, "clip_ratio/low_mean": 0.0029146174609195443, "clip_ratio/low_min": 0.0029146174609195443, "clip_ratio/region_mean": 0.006065310223493725, "completions/clipped_ratio": 0.2, "completions/max_length": 1723.4, "completions/max_terminated_length": 1406.4, "completions/mean_length": 1238.0375, "completions/mean_terminated_length": 1067.039288330078, "completions/min_length": 804.9, "completions/min_terminated_length": 804.9, "entropy": 0.22062662653625012, "epoch": 1.3037037037037038, "frac_reward_zero_std": 0.3, "grad_norm": 0.14793914556503296, "learning_rate": 4.628634775942269e-06, "loss": 0.021900516748428345, "num_tokens": 11159975.0, "reward": 0.6060525143519044, "reward_std": 0.4588416843209416, "rewards/correctness_reward/mean": 0.5225000035017728, "rewards/correctness_reward/std": 0.4323632024228573, "rewards/format_reward/mean": 0.08000000193715096, "rewards/format_reward/std": 0.024030577391386032, "rewards/length_efficiency_reward/mean": 0.0035524999955669047, "rewards/length_efficiency_reward/std": 0.005807241797447205, "sampling/importance_sampling_ratio/max": 0.8419945374131202, "sampling/importance_sampling_ratio/mean": 0.1559647722169757, "sampling/importance_sampling_ratio/min": 0.00016946111873039627, "sampling/sampling_logp_difference/max": 2.8627161502838137, "sampling/sampling_logp_difference/mean": 0.025892662070691585, "step": 880, "step_time": 12.682030203659087 }, { "clip_ratio/high_max": 0.002559301073779352, "clip_ratio/high_mean": 0.002559301073779352, "clip_ratio/low_mean": 0.002745915518607944, "clip_ratio/low_min": 0.002745915518607944, "clip_ratio/region_mean": 0.005305216592387296, "completions/clipped_ratio": 0.275, "completions/max_length": 1844.8, "completions/max_terminated_length": 1576.4, "completions/mean_length": 1357.025, "completions/mean_terminated_length": 1148.3678588867188, "completions/min_length": 755.1, "completions/min_terminated_length": 755.1, "entropy": 0.25810795873403547, "epoch": 1.3185185185185184, "frac_reward_zero_std": 0.45, "grad_norm": 0.06215117126703262, "learning_rate": 4.617857238442764e-06, "loss": 0.0723828375339508, "num_tokens": 11279661.0, "reward": 0.49443813376128676, "reward_std": 0.4618906444869936, "rewards/correctness_reward/mean": 0.42000000178813934, "rewards/correctness_reward/std": 0.4341683030128479, "rewards/format_reward/mean": 0.07250000145286321, "rewards/format_reward/std": 0.029375802725553513, "rewards/length_efficiency_reward/mean": 0.0019381249905563891, "rewards/length_efficiency_reward/std": 0.0034200316295027733, "sampling/importance_sampling_ratio/max": 0.890443866699934, "sampling/importance_sampling_ratio/mean": 0.16853495817631484, "sampling/importance_sampling_ratio/min": 0.00036492240706138547, "sampling/sampling_logp_difference/max": 2.3809892177581786, "sampling/sampling_logp_difference/mean": 0.026819409616291524, "step": 890, "step_time": 13.397584106586873 }, { "clip_ratio/high_max": 0.0033587060490390284, "clip_ratio/high_mean": 0.0033587060490390284, "clip_ratio/low_mean": 0.004619496673694812, "clip_ratio/low_min": 0.004619496673694812, "clip_ratio/region_mean": 0.00797820272273384, "completions/clipped_ratio": 0.125, "completions/max_length": 1709.9, "completions/max_terminated_length": 1542.5, "completions/mean_length": 1184.6125, "completions/mean_terminated_length": 1065.1353637695313, "completions/min_length": 728.1, "completions/min_terminated_length": 728.1, "entropy": 0.2582945663481951, "epoch": 1.3333333333333333, "frac_reward_zero_std": 0.2, "grad_norm": 0.11265179514884949, "learning_rate": 4.606938463567104e-06, "loss": -0.03603248000144958, "num_tokens": 11385674.0, "reward": 0.7530606359243393, "reward_std": 0.34706352557986975, "rewards/correctness_reward/mean": 0.6625000014901161, "rewards/correctness_reward/std": 0.32933457791805265, "rewards/format_reward/mean": 0.08750000186264514, "rewards/format_reward/std": 0.01868535205721855, "rewards/length_efficiency_reward/mean": 0.0030606250362325226, "rewards/length_efficiency_reward/std": 0.0038953334696998354, "sampling/importance_sampling_ratio/max": 1.0380912095308303, "sampling/importance_sampling_ratio/mean": 0.20624214671552182, "sampling/importance_sampling_ratio/min": 9.625350628539309e-06, "sampling/sampling_logp_difference/max": 1.7822023630142212, "sampling/sampling_logp_difference/mean": 0.026992356032133104, "step": 900, "step_time": 12.557231476716698 }, { "clip_ratio/high_max": 0.0026843727682717144, "clip_ratio/high_mean": 0.0026843727682717144, "clip_ratio/low_mean": 0.003603188871056773, "clip_ratio/low_min": 0.003603188871056773, "clip_ratio/region_mean": 0.006287561639328487, "completions/clipped_ratio": 0.2375, "completions/max_length": 1778.0, "completions/max_terminated_length": 1306.7, "completions/mean_length": 1228.9875, "completions/mean_terminated_length": 928.0565673828125, "completions/min_length": 652.4, "completions/min_terminated_length": 652.4, "entropy": 0.2377848188392818, "epoch": 1.348148148148148, "frac_reward_zero_std": 0.2, "grad_norm": 0.5603146553039551, "learning_rate": 4.595879179475379e-06, "loss": -0.03579925894737244, "num_tokens": 11495945.0, "reward": 0.7349575094878673, "reward_std": 0.35726580596528945, "rewards/correctness_reward/mean": 0.6525000017136335, "rewards/correctness_reward/std": 0.32892500460147855, "rewards/format_reward/mean": 0.07625000225380063, "rewards/format_reward/std": 0.025926153361797332, "rewards/length_efficiency_reward/mean": 0.006207499993615783, "rewards/length_efficiency_reward/std": 0.006946719659026712, "sampling/importance_sampling_ratio/max": 1.2663333669304848, "sampling/importance_sampling_ratio/mean": 0.30650049783289435, "sampling/importance_sampling_ratio/min": 0.001370042261078197, "sampling/sampling_logp_difference/max": 2.508310830593109, "sampling/sampling_logp_difference/mean": 0.024790840782225133, "step": 910, "step_time": 13.048589952196926 }, { "clip_ratio/high_max": 0.00212663618294755, "clip_ratio/high_mean": 0.00212663618294755, "clip_ratio/low_mean": 0.004460622230544687, "clip_ratio/low_min": 0.004460622230544687, "clip_ratio/region_mean": 0.0065872584134922365, "completions/clipped_ratio": 0.2875, "completions/max_length": 1672.8, "completions/max_terminated_length": 1306.1, "completions/mean_length": 1211.5875, "completions/mean_terminated_length": 1022.3767883300782, "completions/min_length": 803.8, "completions/min_terminated_length": 803.8, "entropy": 0.2308249158784747, "epoch": 1.362962962962963, "frac_reward_zero_std": 0.25, "grad_norm": 0.013331028632819653, "learning_rate": 4.584680123698069e-06, "loss": -0.05319979786872864, "num_tokens": 11605040.0, "reward": 0.6328299980610609, "reward_std": 0.29014204009436073, "rewards/correctness_reward/mean": 0.5550000015646219, "rewards/correctness_reward/std": 0.26037131547927855, "rewards/format_reward/mean": 0.0712500018067658, "rewards/format_reward/std": 0.027566111460328103, "rewards/length_efficiency_reward/mean": 0.0065800001379102465, "rewards/length_efficiency_reward/std": 0.006142669357359409, "sampling/importance_sampling_ratio/max": 0.5538411036133766, "sampling/importance_sampling_ratio/mean": 0.15590000413358213, "sampling/importance_sampling_ratio/min": 0.000316412749907613, "sampling/sampling_logp_difference/max": 2.1603816390037536, "sampling/sampling_logp_difference/mean": 0.026350138522684575, "step": 920, "step_time": 12.48766271956265 }, { "clip_ratio/high_max": 0.002213004103396088, "clip_ratio/high_mean": 0.002213004103396088, "clip_ratio/low_mean": 0.003719661425566301, "clip_ratio/low_min": 0.003719661425566301, "clip_ratio/region_mean": 0.005932665528962389, "completions/clipped_ratio": 0.2625, "completions/max_length": 1793.4, "completions/max_terminated_length": 1272.1, "completions/mean_length": 1190.3875, "completions/mean_terminated_length": 924.3882263183593, "completions/min_length": 666.0, "completions/min_terminated_length": 666.0, "entropy": 0.22659369092434645, "epoch": 1.3777777777777778, "frac_reward_zero_std": 0.4, "grad_norm": 0.0, "learning_rate": 4.573342043086862e-06, "loss": -0.001831570640206337, "num_tokens": 11711391.0, "reward": 0.6156543909572065, "reward_std": 0.35016259043477477, "rewards/correctness_reward/mean": 0.5350000031292439, "rewards/correctness_reward/std": 0.3124013155698776, "rewards/format_reward/mean": 0.07375000193715095, "rewards/format_reward/std": 0.0336274616420269, "rewards/length_efficiency_reward/mean": 0.0069043751514982436, "rewards/length_efficiency_reward/std": 0.006668158818501979, "sampling/importance_sampling_ratio/max": 0.8837864682078361, "sampling/importance_sampling_ratio/mean": 0.16612715013325213, "sampling/importance_sampling_ratio/min": 9.458641379751498e-05, "sampling/sampling_logp_difference/max": 2.365035665035248, "sampling/sampling_logp_difference/mean": 0.025491106137633324, "step": 930, "step_time": 12.969288874045015 }, { "clip_ratio/high_max": 0.0015527211857261137, "clip_ratio/high_mean": 0.0015527211857261137, "clip_ratio/low_mean": 0.0019024580076802522, "clip_ratio/low_min": 0.0019024580076802522, "clip_ratio/region_mean": 0.0034551791934063656, "completions/clipped_ratio": 0.175, "completions/max_length": 1843.0, "completions/max_terminated_length": 1482.7, "completions/mean_length": 1227.9125, "completions/mean_terminated_length": 1030.7410766601563, "completions/min_length": 743.7, "completions/min_terminated_length": 743.7, "entropy": 0.23153538648039101, "epoch": 1.3925925925925926, "frac_reward_zero_std": 0.55, "grad_norm": 0.05625421926379204, "learning_rate": 4.5618656937648465e-06, "loss": 0.018175390362739564, "num_tokens": 11820320.0, "reward": 0.6010743819177151, "reward_std": 0.3263381006196141, "rewards/correctness_reward/mean": 0.5150000020861626, "rewards/correctness_reward/std": 0.3022254049777985, "rewards/format_reward/mean": 0.08250000216066837, "rewards/format_reward/std": 0.023106744140386583, "rewards/length_efficiency_reward/mean": 0.00357437499333173, "rewards/length_efficiency_reward/std": 0.004765644483268261, "sampling/importance_sampling_ratio/max": 0.8545751191675663, "sampling/importance_sampling_ratio/mean": 0.17365676071494818, "sampling/importance_sampling_ratio/min": 0.0013949861357104964, "sampling/sampling_logp_difference/max": 2.4544323682785034, "sampling/sampling_logp_difference/mean": 0.025336411036550998, "step": 940, "step_time": 13.18983029378578 }, { "clip_ratio/high_max": 0.0035573499917518348, "clip_ratio/high_mean": 0.0035573499917518348, "clip_ratio/low_mean": 0.004580956691643223, "clip_ratio/low_min": 0.004580956691643223, "clip_ratio/region_mean": 0.008138306683395057, "completions/clipped_ratio": 0.3625, "completions/max_length": 1919.0, "completions/max_terminated_length": 1652.6, "completions/mean_length": 1449.8375, "completions/mean_terminated_length": 1191.983349609375, "completions/min_length": 866.9, "completions/min_terminated_length": 866.9, "entropy": 0.24207586012780666, "epoch": 1.4074074074074074, "frac_reward_zero_std": 0.15, "grad_norm": 0.1114610806107521, "learning_rate": 4.550251841076086e-06, "loss": 0.011338778585195542, "num_tokens": 11949611.0, "reward": 0.4959425079636276, "reward_std": 0.5061461256816984, "rewards/correctness_reward/mean": 0.4274999998509884, "rewards/correctness_reward/std": 0.4654659181833267, "rewards/format_reward/mean": 0.06375000122934579, "rewards/format_reward/std": 0.03976476229727268, "rewards/length_efficiency_reward/mean": 0.004692500119563192, "rewards/length_efficiency_reward/std": 0.00534183734562248, "sampling/importance_sampling_ratio/max": 0.6842955987900495, "sampling/importance_sampling_ratio/mean": 0.12434606458991766, "sampling/importance_sampling_ratio/min": 6.896202393136264e-06, "sampling/sampling_logp_difference/max": 2.6388038873672484, "sampling/sampling_logp_difference/mean": 0.02610219269990921, "step": 950, "step_time": 13.96942266812548 }, { "clip_ratio/high_max": 0.003371011260605883, "clip_ratio/high_mean": 0.003371011260605883, "clip_ratio/low_mean": 0.003897259276709519, "clip_ratio/low_min": 0.003897259276709519, "clip_ratio/region_mean": 0.0072682705373154025, "completions/clipped_ratio": 0.15, "completions/max_length": 1741.6, "completions/max_terminated_length": 1564.3, "completions/mean_length": 1124.35, "completions/mean_terminated_length": 985.9235778808594, "completions/min_length": 665.0, "completions/min_terminated_length": 665.0, "entropy": 0.23370868451893329, "epoch": 1.4222222222222223, "frac_reward_zero_std": 0.25, "grad_norm": 0.0, "learning_rate": 4.538501259534581e-06, "loss": -0.04831493496894836, "num_tokens": 12051771.0, "reward": 0.8138475343585014, "reward_std": 0.34259481467306613, "rewards/correctness_reward/mean": 0.7200000047683716, "rewards/correctness_reward/std": 0.31944039165973664, "rewards/format_reward/mean": 0.08500000163912773, "rewards/format_reward/std": 0.01940147690474987, "rewards/length_efficiency_reward/mean": 0.008847500302363188, "rewards/length_efficiency_reward/std": 0.007332965591922402, "sampling/importance_sampling_ratio/max": 0.9769416280090809, "sampling/importance_sampling_ratio/mean": 0.16370523693040012, "sampling/importance_sampling_ratio/min": 0.00015417707240885647, "sampling/sampling_logp_difference/max": 3.6143327355384827, "sampling/sampling_logp_difference/mean": 0.03016998041421175, "step": 960, "step_time": 12.72144062668085 }, { "clip_ratio/high_max": 0.002270848280750215, "clip_ratio/high_mean": 0.002270848280750215, "clip_ratio/low_mean": 0.002348728259676136, "clip_ratio/low_min": 0.002348728259676136, "clip_ratio/region_mean": 0.0046195765404263515, "completions/clipped_ratio": 0.2375, "completions/max_length": 1779.9, "completions/max_terminated_length": 1485.5, "completions/mean_length": 1216.7, "completions/mean_terminated_length": 1010.1189392089843, "completions/min_length": 709.1, "completions/min_terminated_length": 709.1, "entropy": 0.21898404508829117, "epoch": 1.4370370370370371, "frac_reward_zero_std": 0.45, "grad_norm": 0.020159931853413582, "learning_rate": 4.5266147327726165e-06, "loss": 0.009757153689861298, "num_tokens": 12160791.0, "reward": 0.5464500237256289, "reward_std": 0.45378004387021065, "rewards/correctness_reward/mean": 0.46500000543892384, "rewards/correctness_reward/std": 0.4256188780069351, "rewards/format_reward/mean": 0.07625000197440386, "rewards/format_reward/std": 0.027566111460328103, "rewards/length_efficiency_reward/mean": 0.00520000010728836, "rewards/length_efficiency_reward/std": 0.005976194248069078, "sampling/importance_sampling_ratio/max": 0.64504754235968, "sampling/importance_sampling_ratio/mean": 0.13685408886522055, "sampling/importance_sampling_ratio/min": 0.0011249211564063443, "sampling/sampling_logp_difference/max": 4.0627816915512085, "sampling/sampling_logp_difference/mean": 0.0263185553252697, "step": 970, "step_time": 13.04366639945656 }, { "clip_ratio/high_max": 0.0035751662726397626, "clip_ratio/high_mean": 0.0035751662726397626, "clip_ratio/low_mean": 0.002852385799633339, "clip_ratio/low_min": 0.002852385799633339, "clip_ratio/region_mean": 0.006427552072273102, "completions/clipped_ratio": 0.2, "completions/max_length": 1697.7, "completions/max_terminated_length": 1427.9, "completions/mean_length": 1066.175, "completions/mean_terminated_length": 843.8365600585937, "completions/min_length": 565.2, "completions/min_terminated_length": 565.2, "entropy": 0.21213877433910966, "epoch": 1.4518518518518517, "frac_reward_zero_std": 0.2, "grad_norm": 0.09356260299682617, "learning_rate": 4.5145930534885e-06, "loss": -0.006534757465124131, "num_tokens": 12256713.0, "reward": 0.6614181473851204, "reward_std": 0.3315704685170203, "rewards/correctness_reward/mean": 0.5725000023841857, "rewards/correctness_reward/std": 0.299854052066803, "rewards/format_reward/mean": 0.08000000230967999, "rewards/format_reward/std": 0.02739637829363346, "rewards/length_efficiency_reward/mean": 0.00891812490299344, "rewards/length_efficiency_reward/std": 0.008308889949694276, "sampling/importance_sampling_ratio/max": 0.7502260169014334, "sampling/importance_sampling_ratio/mean": 0.12835889630950986, "sampling/importance_sampling_ratio/min": 3.9250925718192774e-05, "sampling/sampling_logp_difference/max": 3.9674753665924074, "sampling/sampling_logp_difference/mean": 0.028327577747404575, "step": 980, "step_time": 12.420807621069253 }, { "clip_ratio/high_max": 0.0026401991897728296, "clip_ratio/high_mean": 0.0026401991897728296, "clip_ratio/low_mean": 0.004291360240313224, "clip_ratio/low_min": 0.004291360240313224, "clip_ratio/region_mean": 0.006931559430086054, "completions/clipped_ratio": 0.125, "completions/max_length": 1721.9, "completions/max_terminated_length": 1641.0, "completions/mean_length": 1190.9875, "completions/mean_terminated_length": 1059.6025024414062, "completions/min_length": 692.2, "completions/min_terminated_length": 692.2, "entropy": 0.22538378685712815, "epoch": 1.4666666666666668, "frac_reward_zero_std": 0.15, "grad_norm": 0.07568373531103134, "learning_rate": 4.5024370233937005e-06, "loss": -0.04884762465953827, "num_tokens": 12363112.0, "reward": 0.8352081567049027, "reward_std": 0.34925237582065166, "rewards/correctness_reward/mean": 0.7375000029802322, "rewards/correctness_reward/std": 0.3272196799516678, "rewards/format_reward/mean": 0.08750000111758709, "rewards/format_reward/std": 0.01569620966911316, "rewards/length_efficiency_reward/mean": 0.010208124830387532, "rewards/length_efficiency_reward/std": 0.007680828438606113, "sampling/importance_sampling_ratio/max": 0.7738640829920769, "sampling/importance_sampling_ratio/mean": 0.1633150763809681, "sampling/importance_sampling_ratio/min": 0.0003216334221633588, "sampling/sampling_logp_difference/max": 2.76413015127182, "sampling/sampling_logp_difference/mean": 0.026055401284247638, "step": 990, "step_time": 12.60168900731951 }, { "clip_ratio/high_max": 0.002675944095244631, "clip_ratio/high_mean": 0.002675944095244631, "clip_ratio/low_mean": 0.004143248905893415, "clip_ratio/low_min": 0.004143248905893415, "clip_ratio/region_mean": 0.0068191930011380466, "completions/clipped_ratio": 0.2375, "completions/max_length": 1768.5, "completions/max_terminated_length": 1512.0, "completions/mean_length": 1298.25, "completions/mean_terminated_length": 1145.3660766601563, "completions/min_length": 823.4, "completions/min_terminated_length": 823.4, "entropy": 0.24021147172898055, "epoch": 1.4814814814814814, "frac_reward_zero_std": 0.3, "grad_norm": 0.001346216187812388, "learning_rate": 4.490147453159386e-06, "loss": -0.009299220144748687, "num_tokens": 12479668.0, "reward": 0.4958599999547005, "reward_std": 0.3579604284372181, "rewards/correctness_reward/mean": 0.41499999836087226, "rewards/correctness_reward/std": 0.3361411988735199, "rewards/format_reward/mean": 0.0762500018812716, "rewards/format_reward/std": 0.022220886126160622, "rewards/length_efficiency_reward/mean": 0.0046099999774014576, "rewards/length_efficiency_reward/std": 0.0045624597871210424, "sampling/importance_sampling_ratio/max": 0.5356515139341355, "sampling/importance_sampling_ratio/mean": 0.1044102530926466, "sampling/importance_sampling_ratio/min": 0.00029526757411986183, "sampling/sampling_logp_difference/max": 3.0100553393363954, "sampling/sampling_logp_difference/mean": 0.026938194036483766, "step": 1000, "step_time": 13.0345803424716 }, { "clip_ratio/high_max": 0.0025053281256987247, "clip_ratio/high_mean": 0.0025053281256987247, "clip_ratio/low_mean": 0.0038723331439541654, "clip_ratio/low_min": 0.0038723331439541654, "clip_ratio/region_mean": 0.00637766126965289, "completions/clipped_ratio": 0.175, "completions/max_length": 1717.6, "completions/max_terminated_length": 1441.6, "completions/mean_length": 1191.35, "completions/mean_terminated_length": 1015.1785827636719, "completions/min_length": 647.1, "completions/min_terminated_length": 647.1, "entropy": 0.21374302776530385, "epoch": 1.4962962962962962, "frac_reward_zero_std": 0.3, "grad_norm": 0.0, "learning_rate": 4.477725162362351e-06, "loss": -0.0543786346912384, "num_tokens": 12586152.0, "reward": 0.730731249973178, "reward_std": 0.25823499229736624, "rewards/correctness_reward/mean": 0.6400000035762787, "rewards/correctness_reward/std": 0.23155785501003265, "rewards/format_reward/mean": 0.08250000216066837, "rewards/format_reward/std": 0.023106744140386583, "rewards/length_efficiency_reward/mean": 0.0082312501501292, "rewards/length_efficiency_reward/std": 0.007771236449480056, "sampling/importance_sampling_ratio/max": 1.0647780228406192, "sampling/importance_sampling_ratio/mean": 0.23999000992625952, "sampling/importance_sampling_ratio/min": 0.0004399564743152951, "sampling/sampling_logp_difference/max": 2.4541333436965944, "sampling/sampling_logp_difference/mean": 0.025377615820616484, "step": 1010, "step_time": 12.80544500015676 }, { "clip_ratio/high_max": 0.002356888953363523, "clip_ratio/high_mean": 0.002356888953363523, "clip_ratio/low_mean": 0.004820121574448421, "clip_ratio/low_min": 0.004820121574448421, "clip_ratio/region_mean": 0.0071770105278119445, "completions/clipped_ratio": 0.2, "completions/max_length": 1695.3, "completions/max_terminated_length": 1450.7, "completions/mean_length": 1223.9625, "completions/mean_terminated_length": 1039.8283447265626, "completions/min_length": 652.2, "completions/min_terminated_length": 652.2, "entropy": 0.22410838915966452, "epoch": 1.511111111111111, "frac_reward_zero_std": 0.25, "grad_norm": 0.3305935561656952, "learning_rate": 4.465170979430372e-06, "loss": -0.06002437472343445, "num_tokens": 12695433.0, "reward": 0.5701068982481956, "reward_std": 0.4890847767703235, "rewards/correctness_reward/mean": 0.4850000023841858, "rewards/correctness_reward/std": 0.4660915195941925, "rewards/format_reward/mean": 0.0800000011920929, "rewards/format_reward/std": 0.02104143500328064, "rewards/length_efficiency_reward/mean": 0.005106874997727573, "rewards/length_efficiency_reward/std": 0.0068001487292349335, "sampling/importance_sampling_ratio/max": 1.068904584646225, "sampling/importance_sampling_ratio/mean": 0.17056417763233184, "sampling/importance_sampling_ratio/min": 0.00011982148039706431, "sampling/sampling_logp_difference/max": 2.2701977729797362, "sampling/sampling_logp_difference/mean": 0.026523995958268643, "step": 1020, "step_time": 12.541018421109765 }, { "clip_ratio/high_max": 0.002425080716784578, "clip_ratio/high_mean": 0.002425080716784578, "clip_ratio/low_mean": 0.0022496946272440254, "clip_ratio/low_min": 0.0022496946272440254, "clip_ratio/region_mean": 0.004674775344028603, "completions/clipped_ratio": 0.275, "completions/max_length": 1923.0, "completions/max_terminated_length": 1590.4, "completions/mean_length": 1354.25, "completions/mean_terminated_length": 1129.0214599609376, "completions/min_length": 749.3, "completions/min_terminated_length": 749.3, "entropy": 0.23671548506245016, "epoch": 1.525925925925926, "frac_reward_zero_std": 0.4, "grad_norm": 0.10383858531713486, "learning_rate": 4.4524857415869515e-06, "loss": -0.03719486892223358, "num_tokens": 12821697.0, "reward": 0.5221300166100263, "reward_std": 0.4807670313864946, "rewards/correctness_reward/mean": 0.4450000010430813, "rewards/correctness_reward/std": 0.4467850595712662, "rewards/format_reward/mean": 0.07250000238418579, "rewards/format_reward/std": 0.032741603627800944, "rewards/length_efficiency_reward/mean": 0.004630000097677112, "rewards/length_efficiency_reward/std": 0.0065120423678308725, "sampling/importance_sampling_ratio/max": 0.9283737991936505, "sampling/importance_sampling_ratio/mean": 0.18920598211698234, "sampling/importance_sampling_ratio/min": 8.872641490142996e-05, "sampling/sampling_logp_difference/max": 2.744346332550049, "sampling/sampling_logp_difference/mean": 0.025229057110846043, "step": 1030, "step_time": 14.110346809960902 }, { "clip_ratio/high_max": 0.002666619428782724, "clip_ratio/high_mean": 0.002666619428782724, "clip_ratio/low_mean": 0.0030272967473138125, "clip_ratio/low_min": 0.0030272967473138125, "clip_ratio/region_mean": 0.005693916176096536, "completions/clipped_ratio": 0.075, "completions/max_length": 1767.0, "completions/max_terminated_length": 1599.7, "completions/mean_length": 1129.9, "completions/mean_terminated_length": 1036.48037109375, "completions/min_length": 668.8, "completions/min_terminated_length": 668.8, "entropy": 0.2263034602627158, "epoch": 1.5407407407407407, "frac_reward_zero_std": 0.35, "grad_norm": 0.07124347984790802, "learning_rate": 4.43967029479549e-06, "loss": -0.03572315275669098, "num_tokens": 12923389.0, "reward": 0.7220268972218037, "reward_std": 0.34638767992146313, "rewards/correctness_reward/mean": 0.6225000023841858, "rewards/correctness_reward/std": 0.33050737977027894, "rewards/format_reward/mean": 0.09250000230967999, "rewards/format_reward/std": 0.01241629347205162, "rewards/length_efficiency_reward/mean": 0.007026875042356551, "rewards/length_efficiency_reward/std": 0.0072998026153072715, "sampling/importance_sampling_ratio/max": 1.0019998505711556, "sampling/importance_sampling_ratio/mean": 0.21750534186139703, "sampling/importance_sampling_ratio/min": 0.002761573229872738, "sampling/sampling_logp_difference/max": 1.951542890071869, "sampling/sampling_logp_difference/mean": 0.026272574439644814, "step": 1040, "step_time": 12.703652764577418 }, { "clip_ratio/high_max": 0.0016911226106458344, "clip_ratio/high_mean": 0.0016911226106458344, "clip_ratio/low_mean": 0.0029820067225955426, "clip_ratio/low_min": 0.0029820067225955426, "clip_ratio/region_mean": 0.004673129333241377, "completions/clipped_ratio": 0.25, "completions/max_length": 1669.4, "completions/max_terminated_length": 1448.6, "completions/mean_length": 1263.65, "completions/mean_terminated_length": 1042.5625, "completions/min_length": 730.9, "completions/min_terminated_length": 730.9, "entropy": 0.2249695524573326, "epoch": 1.5555555555555556, "frac_reward_zero_std": 0.45, "grad_norm": 0.06133605167269707, "learning_rate": 4.426725493702865e-06, "loss": -0.026874253153800966, "num_tokens": 13036549.0, "reward": 0.6684493910521269, "reward_std": 0.38847801703959706, "rewards/correctness_reward/mean": 0.5875000022351742, "rewards/correctness_reward/std": 0.356068828701973, "rewards/format_reward/mean": 0.07500000111758709, "rewards/format_reward/std": 0.026726126670837402, "rewards/length_efficiency_reward/mean": 0.005949374893680215, "rewards/length_efficiency_reward/std": 0.007246681535616517, "sampling/importance_sampling_ratio/max": 0.6346139073371887, "sampling/importance_sampling_ratio/mean": 0.14073075540363789, "sampling/importance_sampling_ratio/min": 0.000711314602085622, "sampling/sampling_logp_difference/max": 2.3926549673080446, "sampling/sampling_logp_difference/mean": 0.023596937116235495, "step": 1050, "step_time": 12.492674534860999 }, { "clip_ratio/high_max": 0.0031552410728181712, "clip_ratio/high_mean": 0.0031552410728181712, "clip_ratio/low_mean": 0.003952977497829124, "clip_ratio/low_min": 0.003952977497829124, "clip_ratio/region_mean": 0.007108218570647295, "completions/clipped_ratio": 0.2, "completions/max_length": 1809.7, "completions/max_terminated_length": 1395.6, "completions/mean_length": 1200.8375, "completions/mean_terminated_length": 966.6675109863281, "completions/min_length": 666.9, "completions/min_terminated_length": 666.9, "entropy": 0.2532115889713168, "epoch": 1.5703703703703704, "frac_reward_zero_std": 0.3, "grad_norm": 0.24033764004707336, "learning_rate": 4.4136522015824405e-06, "loss": 0.03623269200325012, "num_tokens": 13145212.0, "reward": 0.657163770776242, "reward_std": 0.451486940542236, "rewards/correctness_reward/mean": 0.5725000038743019, "rewards/correctness_reward/std": 0.4228206604719162, "rewards/format_reward/mean": 0.08000000156462192, "rewards/format_reward/std": 0.02474670223891735, "rewards/length_efficiency_reward/mean": 0.004663750051986426, "rewards/length_efficiency_reward/std": 0.005942460894584656, "sampling/importance_sampling_ratio/max": 0.5033138547092676, "sampling/importance_sampling_ratio/mean": 0.09764478877186775, "sampling/importance_sampling_ratio/min": 0.0010128589451028347, "sampling/sampling_logp_difference/max": 2.669939947128296, "sampling/sampling_logp_difference/mean": 0.0266396876424551, "step": 1060, "step_time": 13.091554485540836 }, { "clip_ratio/high_max": 0.0016988781106192618, "clip_ratio/high_mean": 0.0016988781106192618, "clip_ratio/low_mean": 0.002725660818396136, "clip_ratio/low_min": 0.002725660818396136, "clip_ratio/region_mean": 0.004424538929015398, "completions/clipped_ratio": 0.225, "completions/max_length": 1908.1, "completions/max_terminated_length": 1537.4, "completions/mean_length": 1301.9625, "completions/mean_terminated_length": 979.8541687011718, "completions/min_length": 816.4, "completions/min_terminated_length": 611.6, "entropy": 0.25553709277883174, "epoch": 1.585185185185185, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 4.400451290276494e-06, "loss": -0.003675276041030884, "num_tokens": 13263725.0, "reward": 0.5741062641143799, "reward_std": 0.4350778728723526, "rewards/correctness_reward/mean": 0.4924999997019768, "rewards/correctness_reward/std": 0.41864589750766756, "rewards/format_reward/mean": 0.0775000013411045, "rewards/format_reward/std": 0.015319551154971123, "rewards/length_efficiency_reward/mean": 0.0041062499687541274, "rewards/length_efficiency_reward/std": 0.004912064841482789, "sampling/importance_sampling_ratio/max": 0.861373059079051, "sampling/importance_sampling_ratio/mean": 0.15762675274163485, "sampling/importance_sampling_ratio/min": 0.00021352346611820395, "sampling/sampling_logp_difference/max": 2.062123250961304, "sampling/sampling_logp_difference/mean": 0.02606740277260542, "step": 1070, "step_time": 13.743233086541295 }, { "clip_ratio/high_max": 0.0023856259751482865, "clip_ratio/high_mean": 0.0023856259751482865, "clip_ratio/low_mean": 0.004565997081226669, "clip_ratio/low_min": 0.004565997081226669, "clip_ratio/region_mean": 0.006951623056374956, "completions/clipped_ratio": 0.25, "completions/max_length": 1815.4, "completions/max_terminated_length": 1497.9, "completions/mean_length": 1229.1375, "completions/mean_terminated_length": 1031.9713287353516, "completions/min_length": 688.6, "completions/min_terminated_length": 688.6, "entropy": 0.2360273303464055, "epoch": 1.6, "frac_reward_zero_std": 0.2, "grad_norm": 0.0, "learning_rate": 4.387123640138072e-06, "loss": -0.040790256857872007, "num_tokens": 13373868.0, "reward": 0.633306885883212, "reward_std": 0.47155295591801405, "rewards/correctness_reward/mean": 0.5500000029802322, "rewards/correctness_reward/std": 0.43810252845287323, "rewards/format_reward/mean": 0.07500000167638063, "rewards/format_reward/std": 0.028659677878022195, "rewards/length_efficiency_reward/mean": 0.008306875126436353, "rewards/length_efficiency_reward/std": 0.008556123031303287, "sampling/importance_sampling_ratio/max": 1.4020732851698994, "sampling/importance_sampling_ratio/mean": 0.2855456732911989, "sampling/importance_sampling_ratio/min": 0.0006058281148398237, "sampling/sampling_logp_difference/max": 2.883931404352188, "sampling/sampling_logp_difference/mean": 0.025402953382581472, "step": 1080, "step_time": 13.202527128066867 }, { "clip_ratio/high_max": 0.003920698454021476, "clip_ratio/high_mean": 0.003920698454021476, "clip_ratio/low_mean": 0.004517629567999393, "clip_ratio/low_min": 0.004517629567999393, "clip_ratio/region_mean": 0.00843832802202087, "completions/clipped_ratio": 0.075, "completions/max_length": 1658.5, "completions/max_terminated_length": 1573.1, "completions/mean_length": 1038.025, "completions/mean_terminated_length": 977.8593017578125, "completions/min_length": 560.1, "completions/min_terminated_length": 560.1, "entropy": 0.24534560106694697, "epoch": 1.6148148148148147, "frac_reward_zero_std": 0.15, "grad_norm": 0.409298837184906, "learning_rate": 4.373670139972287e-06, "loss": -0.011986471712589264, "num_tokens": 13467534.0, "reward": 0.8102018857374788, "reward_std": 0.3195065988227725, "rewards/correctness_reward/mean": 0.7100000035017728, "rewards/correctness_reward/std": 0.3006440244615078, "rewards/format_reward/mean": 0.09250000193715095, "rewards/format_reward/std": 0.013340126723051071, "rewards/length_efficiency_reward/mean": 0.007701874943450093, "rewards/length_efficiency_reward/std": 0.008060423610731959, "sampling/importance_sampling_ratio/max": 1.3121537849307061, "sampling/importance_sampling_ratio/mean": 0.2262399010360241, "sampling/importance_sampling_ratio/min": 0.0004521532454532462, "sampling/sampling_logp_difference/max": 2.189663600921631, "sampling/sampling_logp_difference/mean": 0.028979203291237356, "step": 1090, "step_time": 12.161589234042912 }, { "clip_ratio/high_max": 0.002132980928581674, "clip_ratio/high_mean": 0.002132980928581674, "clip_ratio/low_mean": 0.005068092266446911, "clip_ratio/low_min": 0.005068092266446911, "clip_ratio/region_mean": 0.007201073195028584, "completions/clipped_ratio": 0.2, "completions/max_length": 1731.5, "completions/max_terminated_length": 1287.8, "completions/mean_length": 1101.7375, "completions/mean_terminated_length": 850.0475036621094, "completions/min_length": 528.8, "completions/min_terminated_length": 528.8, "entropy": 0.23324970994144678, "epoch": 1.6296296296296298, "frac_reward_zero_std": 0.3, "grad_norm": 0.2983299195766449, "learning_rate": 4.360091686977035e-06, "loss": -0.004470175877213478, "num_tokens": 13566841.0, "reward": 0.6566581308841706, "reward_std": 0.5379946342669427, "rewards/correctness_reward/mean": 0.572500005364418, "rewards/correctness_reward/std": 0.5090720683336258, "rewards/format_reward/mean": 0.08000000156462192, "rewards/format_reward/std": 0.02474670223891735, "rewards/length_efficiency_reward/mean": 0.004158125043613836, "rewards/length_efficiency_reward/std": 0.006416575843468308, "sampling/importance_sampling_ratio/max": 0.6644882097840309, "sampling/importance_sampling_ratio/mean": 0.18882915256544947, "sampling/importance_sampling_ratio/min": 0.0005785952867881861, "sampling/sampling_logp_difference/max": 1.9960147500038148, "sampling/sampling_logp_difference/mean": 0.02613520696759224, "step": 1100, "step_time": 12.574378504045308 }, { "clip_ratio/high_max": 0.002130964255775325, "clip_ratio/high_mean": 0.002130964255775325, "clip_ratio/low_mean": 0.002856107958359644, "clip_ratio/low_min": 0.002856107958359644, "clip_ratio/region_mean": 0.004987072214134969, "completions/clipped_ratio": 0.3375, "completions/max_length": 2047.4, "completions/max_terminated_length": 1717.3, "completions/mean_length": 1495.75, "completions/mean_terminated_length": 1262.0407348632812, "completions/min_length": 925.8, "completions/min_terminated_length": 925.8, "entropy": 0.24677341040223838, "epoch": 1.6444444444444444, "frac_reward_zero_std": 0.35, "grad_norm": 0.0, "learning_rate": 4.346389186683171e-06, "loss": -0.024903422594070433, "num_tokens": 13701985.0, "reward": 0.4553837548941374, "reward_std": 0.4847060784697533, "rewards/correctness_reward/mean": 0.38500000461935996, "rewards/correctness_reward/std": 0.44958587512373926, "rewards/format_reward/mean": 0.06750000277534127, "rewards/format_reward/std": 0.03854937218129635, "rewards/length_efficiency_reward/mean": 0.0028837499325163663, "rewards/length_efficiency_reward/std": 0.003354294179007411, "sampling/importance_sampling_ratio/max": 0.5410111136734486, "sampling/importance_sampling_ratio/mean": 0.10161166023463011, "sampling/importance_sampling_ratio/min": 0.000422440155489312, "sampling/sampling_logp_difference/max": 1.9097564816474915, "sampling/sampling_logp_difference/mean": 0.024071687646210194, "step": 1110, "step_time": 14.859250509925186 }, { "clip_ratio/high_max": 0.0023730993270874023, "clip_ratio/high_mean": 0.0023730993270874023, "clip_ratio/low_mean": 0.002186174562666565, "clip_ratio/low_min": 0.002186174562666565, "clip_ratio/region_mean": 0.0045592738897539675, "completions/clipped_ratio": 0.15, "completions/max_length": 1562.2, "completions/max_terminated_length": 1282.7, "completions/mean_length": 1091.775, "completions/mean_terminated_length": 929.4, "completions/min_length": 594.8, "completions/min_terminated_length": 594.8, "entropy": 0.22461608378216624, "epoch": 1.6592592592592592, "frac_reward_zero_std": 0.4, "grad_norm": 0.24307198822498322, "learning_rate": 4.332563552894116e-06, "loss": -0.005553311109542847, "num_tokens": 13802963.0, "reward": 0.8003756172955037, "reward_std": 0.2495164749212563, "rewards/correctness_reward/mean": 0.7075000017881393, "rewards/correctness_reward/std": 0.2277834326028824, "rewards/format_reward/mean": 0.08500000126659871, "rewards/format_reward/std": 0.01603567600250244, "rewards/length_efficiency_reward/mean": 0.007875624834559858, "rewards/length_efficiency_reward/std": 0.006697757542133332, "sampling/importance_sampling_ratio/max": 0.9296635910868645, "sampling/importance_sampling_ratio/mean": 0.21076292730867863, "sampling/importance_sampling_ratio/min": 0.0007604880956932902, "sampling/sampling_logp_difference/max": 2.656423032283783, "sampling/sampling_logp_difference/mean": 0.02521426845341921, "step": 1120, "step_time": 11.767773431912065 }, { "clip_ratio/high_max": 0.00257046935003018, "clip_ratio/high_mean": 0.00257046935003018, "clip_ratio/low_mean": 0.0023450421009329147, "clip_ratio/low_min": 0.0023450421009329147, "clip_ratio/region_mean": 0.004915511450963095, "completions/clipped_ratio": 0.2, "completions/max_length": 1623.6, "completions/max_terminated_length": 1235.3, "completions/mean_length": 1106.675, "completions/mean_terminated_length": 774.3357177734375, "completions/min_length": 709.6, "completions/min_terminated_length": 504.8, "entropy": 0.23286016285419464, "epoch": 1.674074074074074, "frac_reward_zero_std": 0.4, "grad_norm": 0.007141779642552137, "learning_rate": 4.318615707624914e-06, "loss": -0.14593206644058226, "num_tokens": 13903681.0, "reward": 0.682891283929348, "reward_std": 0.34611494913697244, "rewards/correctness_reward/mean": 0.5975000038743019, "rewards/correctness_reward/std": 0.32733442485332487, "rewards/format_reward/mean": 0.08000000156462192, "rewards/format_reward/std": 0.01405625157058239, "rewards/length_efficiency_reward/mean": 0.0053912501083686944, "rewards/length_efficiency_reward/std": 0.005580474110320211, "sampling/importance_sampling_ratio/max": 1.25939132720232, "sampling/importance_sampling_ratio/mean": 0.2683190483599901, "sampling/importance_sampling_ratio/min": 0.0013844450140084064, "sampling/sampling_logp_difference/max": 2.2006312727928163, "sampling/sampling_logp_difference/mean": 0.025875417329370976, "step": 1130, "step_time": 12.186231566406786 }, { "clip_ratio/high_max": 0.0024925362049543764, "clip_ratio/high_mean": 0.0024925362049543764, "clip_ratio/low_mean": 0.003272091195685789, "clip_ratio/low_min": 0.003272091195685789, "clip_ratio/region_mean": 0.005764627400640166, "completions/clipped_ratio": 0.175, "completions/max_length": 1797.0, "completions/max_terminated_length": 1508.8, "completions/mean_length": 1265.875, "completions/mean_terminated_length": 1088.71787109375, "completions/min_length": 694.1, "completions/min_terminated_length": 694.1, "entropy": 0.23432865142822265, "epoch": 1.6888888888888889, "frac_reward_zero_std": 0.3, "grad_norm": 0.01664021611213684, "learning_rate": 4.304546581040752e-06, "loss": -0.02581920325756073, "num_tokens": 14017271.0, "reward": 0.7149837598204613, "reward_std": 0.4252714298199862, "rewards/correctness_reward/mean": 0.6275000028312206, "rewards/correctness_reward/std": 0.4029359698295593, "rewards/format_reward/mean": 0.08250000216066837, "rewards/format_reward/std": 0.023106744140386583, "rewards/length_efficiency_reward/mean": 0.004983750125393272, "rewards/length_efficiency_reward/std": 0.0037695899372920394, "sampling/importance_sampling_ratio/max": 0.6889152195304632, "sampling/importance_sampling_ratio/mean": 0.15939308740198613, "sampling/importance_sampling_ratio/min": 0.0018538681106065268, "sampling/sampling_logp_difference/max": 1.9726284980773925, "sampling/sampling_logp_difference/mean": 0.0240038825199008, "step": 1140, "step_time": 13.179491471871733 }, { "clip_ratio/high_max": 0.002920125630043913, "clip_ratio/high_mean": 0.002920125630043913, "clip_ratio/low_mean": 0.004885041079251096, "clip_ratio/low_min": 0.004885041079251096, "clip_ratio/region_mean": 0.007805166709295009, "completions/clipped_ratio": 0.2125, "completions/max_length": 1949.5, "completions/max_terminated_length": 1803.9, "completions/mean_length": 1437.05, "completions/mean_terminated_length": 1262.3597839355468, "completions/min_length": 841.6, "completions/min_terminated_length": 841.6, "entropy": 0.26989957559853794, "epoch": 1.7037037037037037, "frac_reward_zero_std": 0.15, "grad_norm": 0.1149311512708664, "learning_rate": 4.29035711139492e-06, "loss": -0.05395635962486267, "num_tokens": 14143659.0, "reward": 0.6143456518650054, "reward_std": 0.5206234838813544, "rewards/correctness_reward/mean": 0.5325000047683716, "rewards/correctness_reward/std": 0.4923641115427017, "rewards/format_reward/mean": 0.07875000163912774, "rewards/format_reward/std": 0.02920606955885887, "rewards/length_efficiency_reward/mean": 0.0030956249887822197, "rewards/length_efficiency_reward/std": 0.0045535235665738584, "sampling/importance_sampling_ratio/max": 1.0027844697237014, "sampling/importance_sampling_ratio/mean": 0.1766265019774437, "sampling/importance_sampling_ratio/min": 9.835383916652063e-05, "sampling/sampling_logp_difference/max": 2.919190263748169, "sampling/sampling_logp_difference/mean": 0.02653575912117958, "step": 1150, "step_time": 14.073390201106667 }, { "clip_ratio/high_max": 0.0026527944632107393, "clip_ratio/high_mean": 0.0026527944632107393, "clip_ratio/low_mean": 0.004308339982526377, "clip_ratio/low_min": 0.004308339982526377, "clip_ratio/region_mean": 0.006961134445737116, "completions/clipped_ratio": 0.1, "completions/max_length": 1566.4, "completions/max_terminated_length": 1368.2, "completions/mean_length": 1032.6, "completions/mean_terminated_length": 941.421435546875, "completions/min_length": 603.1, "completions/min_terminated_length": 603.1, "entropy": 0.2572627430781722, "epoch": 1.7185185185185186, "frac_reward_zero_std": 0.2, "grad_norm": 0.03697351738810539, "learning_rate": 4.276048244966245e-06, "loss": -0.0209475040435791, "num_tokens": 14235899.0, "reward": 0.787784384470433, "reward_std": 0.3318345669656992, "rewards/correctness_reward/mean": 0.692500002682209, "rewards/correctness_reward/std": 0.31301194727420806, "rewards/format_reward/mean": 0.09000000171363354, "rewards/format_reward/std": 0.01405625157058239, "rewards/length_efficiency_reward/mean": 0.00528437492903322, "rewards/length_efficiency_reward/std": 0.007246356504037976, "sampling/importance_sampling_ratio/max": 0.740558360517025, "sampling/importance_sampling_ratio/mean": 0.1678544357419014, "sampling/importance_sampling_ratio/min": 0.00016876699082786218, "sampling/sampling_logp_difference/max": 2.56108934879303, "sampling/sampling_logp_difference/mean": 0.027722511254251004, "step": 1160, "step_time": 11.668311400897801 }, { "clip_ratio/high_max": 0.0019499406742397695, "clip_ratio/high_mean": 0.0019499406742397695, "clip_ratio/low_mean": 0.0028572867333423345, "clip_ratio/low_min": 0.0028572867333423345, "clip_ratio/region_mean": 0.004807227407582104, "completions/clipped_ratio": 0.2, "completions/max_length": 1787.8, "completions/max_terminated_length": 1383.2, "completions/mean_length": 1290.0625, "completions/mean_terminated_length": 1085.9333374023438, "completions/min_length": 767.2, "completions/min_terminated_length": 767.2, "entropy": 0.24565661447122694, "epoch": 1.7333333333333334, "frac_reward_zero_std": 0.4, "grad_norm": 0.13527485728263855, "learning_rate": 4.26162093599598e-06, "loss": -0.010071693360805512, "num_tokens": 14352280.0, "reward": 0.5827412765473128, "reward_std": 0.4868491898290813, "rewards/correctness_reward/mean": 0.4974999986588955, "rewards/correctness_reward/std": 0.45875688791275027, "rewards/format_reward/mean": 0.08000000156462192, "rewards/format_reward/std": 0.025293876975774766, "rewards/length_efficiency_reward/mean": 0.005241249979007989, "rewards/length_efficiency_reward/std": 0.005346779944375157, "sampling/importance_sampling_ratio/max": 1.0448753580451011, "sampling/importance_sampling_ratio/mean": 0.20166772045195103, "sampling/importance_sampling_ratio/min": 0.0004892740926152328, "sampling/sampling_logp_difference/max": 1.590196430683136, "sampling/sampling_logp_difference/mean": 0.024153241235762835, "step": 1170, "step_time": 13.157204005867243 }, { "clip_ratio/high_max": 0.0018614347703987733, "clip_ratio/high_mean": 0.0018614347703987733, "clip_ratio/low_mean": 0.0035800895479042084, "clip_ratio/low_min": 0.0035800895479042084, "clip_ratio/region_mean": 0.005441524318302982, "completions/clipped_ratio": 0.15, "completions/max_length": 1638.0, "completions/max_terminated_length": 1408.9, "completions/mean_length": 1193.375, "completions/mean_terminated_length": 1040.9381103515625, "completions/min_length": 697.8, "completions/min_terminated_length": 697.8, "entropy": 0.24551931992173195, "epoch": 1.748148148148148, "frac_reward_zero_std": 0.35, "grad_norm": 0.4721626937389374, "learning_rate": 4.247076146624175e-06, "loss": -0.06086875200271606, "num_tokens": 14459570.0, "reward": 0.7078744024038315, "reward_std": 0.46938770771957933, "rewards/correctness_reward/mean": 0.6200000017881393, "rewards/correctness_reward/std": 0.44738735556602477, "rewards/format_reward/mean": 0.08500000238418579, "rewards/format_reward/std": 0.02239061929285526, "rewards/length_efficiency_reward/mean": 0.0028743749659042805, "rewards/length_efficiency_reward/std": 0.003868604393210262, "sampling/importance_sampling_ratio/max": 1.0889061000198126, "sampling/importance_sampling_ratio/mean": 0.1958835244178772, "sampling/importance_sampling_ratio/min": 0.0006692374823614955, "sampling/sampling_logp_difference/max": 1.8731067657470704, "sampling/sampling_logp_difference/mean": 0.02588374987244606, "step": 1180, "step_time": 12.249032367859035 }, { "clip_ratio/high_max": 0.0017451241503295023, "clip_ratio/high_mean": 0.0017451241503295023, "clip_ratio/low_mean": 0.0033219380478840322, "clip_ratio/low_min": 0.0033219380478840322, "clip_ratio/region_mean": 0.005067062198213535, "completions/clipped_ratio": 0.2625, "completions/max_length": 1723.7, "completions/max_terminated_length": 1347.8, "completions/mean_length": 1173.7125, "completions/mean_terminated_length": 939.571435546875, "completions/min_length": 688.0, "completions/min_terminated_length": 688.0, "entropy": 0.2338933838531375, "epoch": 1.762962962962963, "frac_reward_zero_std": 0.4, "grad_norm": 0.0821012556552887, "learning_rate": 4.232414846825504e-06, "loss": 0.0044815119355916975, "num_tokens": 14565335.0, "reward": 0.5798443980515003, "reward_std": 0.3371840993408114, "rewards/correctness_reward/mean": 0.5, "rewards/correctness_reward/std": 0.3099078319966793, "rewards/format_reward/mean": 0.07500000121071934, "rewards/format_reward/std": 0.024030577391386032, "rewards/length_efficiency_reward/mean": 0.0048443748499266805, "rewards/length_efficiency_reward/std": 0.006303601298714057, "sampling/importance_sampling_ratio/max": 1.1614839166402817, "sampling/importance_sampling_ratio/mean": 0.2759202405810356, "sampling/importance_sampling_ratio/min": 0.003031315264524892, "sampling/sampling_logp_difference/max": 1.7578369855880738, "sampling/sampling_logp_difference/mean": 0.023914423678070305, "step": 1190, "step_time": 12.717639337014408 }, { "clip_ratio/high_max": 0.0029943405941594394, "clip_ratio/high_mean": 0.0029943405941594394, "clip_ratio/low_mean": 0.004710773861734197, "clip_ratio/low_min": 0.004710773861734197, "clip_ratio/region_mean": 0.007705114455893636, "completions/clipped_ratio": 0.175, "completions/max_length": 1811.5, "completions/max_terminated_length": 1449.4, "completions/mean_length": 1300.2375, "completions/mean_terminated_length": 1140.4059692382812, "completions/min_length": 772.2, "completions/min_terminated_length": 772.2, "entropy": 0.26594371218234303, "epoch": 1.7777777777777777, "frac_reward_zero_std": 0.15, "grad_norm": 0.08315679430961609, "learning_rate": 4.217638014344585e-06, "loss": -0.010225355625152588, "num_tokens": 14681058.0, "reward": 0.6873906448483467, "reward_std": 0.4254252282204106, "rewards/correctness_reward/mean": 0.6025000032037496, "rewards/correctness_reward/std": 0.40369699001312254, "rewards/format_reward/mean": 0.0825000025331974, "rewards/format_reward/std": 0.027019719779491424, "rewards/length_efficiency_reward/mean": 0.0023906249602987373, "rewards/length_efficiency_reward/std": 0.0036544198759656866, "sampling/importance_sampling_ratio/max": 0.5443129241466522, "sampling/importance_sampling_ratio/mean": 0.10285376906394958, "sampling/importance_sampling_ratio/min": 0.0006822993555260837, "sampling/sampling_logp_difference/max": 2.4579389452934266, "sampling/sampling_logp_difference/mean": 0.026877432502806185, "step": 1200, "step_time": 13.173143443651497 }, { "clip_ratio/high_max": 0.002649838413344696, "clip_ratio/high_mean": 0.002649838413344696, "clip_ratio/low_mean": 0.003263315340154804, "clip_ratio/low_min": 0.003263315340154804, "clip_ratio/region_mean": 0.005913153753499501, "completions/clipped_ratio": 0.1375, "completions/max_length": 1654.4, "completions/max_terminated_length": 1502.7, "completions/mean_length": 1170.2125, "completions/mean_terminated_length": 1050.289892578125, "completions/min_length": 690.7, "completions/min_terminated_length": 690.7, "entropy": 0.23508837586268783, "epoch": 1.7925925925925927, "frac_reward_zero_std": 0.25, "grad_norm": 0.010696638375520706, "learning_rate": 4.202746634630772e-06, "loss": -0.016290290653705596, "num_tokens": 14786631.0, "reward": 0.7964287752285599, "reward_std": 0.26878545065410436, "rewards/correctness_reward/mean": 0.7074999999254942, "rewards/correctness_reward/std": 0.24484772309660913, "rewards/format_reward/mean": 0.08500000163912773, "rewards/format_reward/std": 0.019948651641607286, "rewards/length_efficiency_reward/mean": 0.003928749996703118, "rewards/length_efficiency_reward/std": 0.004590183624532074, "sampling/importance_sampling_ratio/max": 0.8146557666361331, "sampling/importance_sampling_ratio/mean": 0.18710540495812894, "sampling/importance_sampling_ratio/min": 0.00732751620162162, "sampling/sampling_logp_difference/max": 2.001113271713257, "sampling/sampling_logp_difference/mean": 0.0255854744464159, "step": 1210, "step_time": 12.319071148801594 }, { "clip_ratio/high_max": 0.003256207035155967, "clip_ratio/high_mean": 0.003256207035155967, "clip_ratio/low_mean": 0.00508106552879326, "clip_ratio/low_min": 0.00508106552879326, "clip_ratio/region_mean": 0.008337272563949227, "completions/clipped_ratio": 0.125, "completions/max_length": 1436.7, "completions/max_terminated_length": 1123.0, "completions/mean_length": 998.5125, "completions/mean_terminated_length": 848.5107299804688, "completions/min_length": 593.3, "completions/min_terminated_length": 593.3, "entropy": 0.23470116686075926, "epoch": 1.8074074074074074, "frac_reward_zero_std": 0.15, "grad_norm": 0.13159742951393127, "learning_rate": 4.187741700772437e-06, "loss": -0.04366266131401062, "num_tokens": 14878496.0, "reward": 0.8463206231594086, "reward_std": 0.3454766800161451, "rewards/correctness_reward/mean": 0.7500000059604645, "rewards/correctness_reward/std": 0.32004271149635316, "rewards/format_reward/mean": 0.08750000223517418, "rewards/format_reward/std": 0.017761518806219102, "rewards/length_efficiency_reward/mean": 0.008820625068619848, "rewards/length_efficiency_reward/std": 0.008689313381910323, "sampling/importance_sampling_ratio/max": 0.9384044945240021, "sampling/importance_sampling_ratio/mean": 0.2262361105531454, "sampling/importance_sampling_ratio/min": 0.0021911159115134637, "sampling/sampling_logp_difference/max": 1.9547487258911134, "sampling/sampling_logp_difference/mean": 0.028235669434070586, "step": 1220, "step_time": 11.112048153951765 }, { "clip_ratio/high_max": 0.003095082868821919, "clip_ratio/high_mean": 0.003095082868821919, "clip_ratio/low_mean": 0.002917089534457773, "clip_ratio/low_min": 0.002917089534457773, "clip_ratio/region_mean": 0.006012172403279692, "completions/clipped_ratio": 0.2, "completions/max_length": 1675.1, "completions/max_terminated_length": 1219.4, "completions/mean_length": 1048.65, "completions/mean_terminated_length": 783.525, "completions/min_length": 525.5, "completions/min_terminated_length": 525.5, "entropy": 0.21151028387248516, "epoch": 1.8222222222222222, "frac_reward_zero_std": 0.4, "grad_norm": 0.0966784656047821, "learning_rate": 4.172624213430743e-06, "loss": -0.023925477266311647, "num_tokens": 14973284.0, "reward": 0.5977031450718642, "reward_std": 0.40857162829488514, "rewards/correctness_reward/mean": 0.5100000016391277, "rewards/correctness_reward/std": 0.3807337164878845, "rewards/format_reward/mean": 0.0800000011920929, "rewards/format_reward/std": 0.021380901336669922, "rewards/length_efficiency_reward/mean": 0.007703125005355105, "rewards/length_efficiency_reward/std": 0.0075676622800529, "sampling/importance_sampling_ratio/max": 1.1030135110020638, "sampling/importance_sampling_ratio/mean": 0.24257346838712693, "sampling/importance_sampling_ratio/min": 0.0017004850298690144, "sampling/sampling_logp_difference/max": 2.2243749499320984, "sampling/sampling_logp_difference/mean": 0.025001751258969307, "step": 1230, "step_time": 12.216720166802407 }, { "clip_ratio/high_max": 0.0024283431368530726, "clip_ratio/high_mean": 0.0024283431368530726, "clip_ratio/low_mean": 0.0030518228624714537, "clip_ratio/low_min": 0.0030518228624714537, "clip_ratio/region_mean": 0.005480165999324526, "completions/clipped_ratio": 0.1875, "completions/max_length": 1788.7, "completions/max_terminated_length": 1613.0, "completions/mean_length": 1247.7375, "completions/mean_terminated_length": 1101.8107177734375, "completions/min_length": 724.1, "completions/min_terminated_length": 724.1, "entropy": 0.2439079715870321, "epoch": 1.837037037037037, "frac_reward_zero_std": 0.35, "grad_norm": 0.0, "learning_rate": 4.157395180772911e-06, "loss": 0.04973750412464142, "num_tokens": 15085475.0, "reward": 0.6497412692755461, "reward_std": 0.416461697826162, "rewards/correctness_reward/mean": 0.5625000022351742, "rewards/correctness_reward/std": 0.39015171229839324, "rewards/format_reward/mean": 0.08125000186264515, "rewards/format_reward/std": 0.024200310558080675, "rewards/length_efficiency_reward/mean": 0.005991249927319586, "rewards/length_efficiency_reward/std": 0.006551528256386519, "sampling/importance_sampling_ratio/max": 0.8348191320896149, "sampling/importance_sampling_ratio/mean": 0.19028282649815081, "sampling/importance_sampling_ratio/min": 0.00597424152883832, "sampling/sampling_logp_difference/max": 1.6798001170158385, "sampling/sampling_logp_difference/mean": 0.025537857413291933, "step": 1240, "step_time": 13.088018954638391 }, { "clip_ratio/high_max": 0.002330583511502482, "clip_ratio/high_mean": 0.002330583511502482, "clip_ratio/low_mean": 0.0038845243863761423, "clip_ratio/low_min": 0.0038845243863761423, "clip_ratio/region_mean": 0.006215107897878625, "completions/clipped_ratio": 0.1875, "completions/max_length": 1733.9, "completions/max_terminated_length": 1491.0, "completions/mean_length": 1246.175, "completions/mean_terminated_length": 1091.5153686523438, "completions/min_length": 756.6, "completions/min_terminated_length": 756.6, "entropy": 0.23747810907661915, "epoch": 1.8518518518518519, "frac_reward_zero_std": 0.35, "grad_norm": 0.012019011192023754, "learning_rate": 4.142055618404986e-06, "loss": -0.015491670370101929, "num_tokens": 15199349.0, "reward": 0.5242281424812972, "reward_std": 0.46155038559809325, "rewards/correctness_reward/mean": 0.43750000447034837, "rewards/correctness_reward/std": 0.4353061646223068, "rewards/format_reward/mean": 0.08125000149011612, "rewards/format_reward/std": 0.02457696907222271, "rewards/length_efficiency_reward/mean": 0.005478124925139128, "rewards/length_efficiency_reward/std": 0.005637657553597819, "sampling/importance_sampling_ratio/max": 0.7067959282547236, "sampling/importance_sampling_ratio/mean": 0.12394883544184268, "sampling/importance_sampling_ratio/min": 0.00016908508559936307, "sampling/sampling_logp_difference/max": 2.3144150495529177, "sampling/sampling_logp_difference/mean": 0.02619993854314089, "step": 1250, "step_time": 12.840315380506217 }, { "clip_ratio/high_max": 0.002848674049891997, "clip_ratio/high_mean": 0.002848674049891997, "clip_ratio/low_mean": 0.004051332696690224, "clip_ratio/low_min": 0.004051332696690224, "clip_ratio/region_mean": 0.006900006746582222, "completions/clipped_ratio": 0.15, "completions/max_length": 1539.2, "completions/max_terminated_length": 1198.1, "completions/mean_length": 1025.1125, "completions/mean_terminated_length": 841.2996520996094, "completions/min_length": 567.7, "completions/min_terminated_length": 567.7, "entropy": 0.2095701106823981, "epoch": 1.8666666666666667, "frac_reward_zero_std": 0.3, "grad_norm": 0.08975726366043091, "learning_rate": 4.126606549304109e-06, "loss": -0.049739676713943484, "num_tokens": 15292814.0, "reward": 0.6475206464529037, "reward_std": 0.47874907599762084, "rewards/correctness_reward/mean": 0.5575000017881393, "rewards/correctness_reward/std": 0.4571537673473358, "rewards/format_reward/mean": 0.08500000163912773, "rewards/format_reward/std": 0.01940147690474987, "rewards/length_efficiency_reward/mean": 0.005020624957978725, "rewards/length_efficiency_reward/std": 0.006489689601585269, "sampling/importance_sampling_ratio/max": 0.9434191763401032, "sampling/importance_sampling_ratio/mean": 0.24265309162437915, "sampling/importance_sampling_ratio/min": 0.00032121986077982, "sampling/sampling_logp_difference/max": 2.261965310573578, "sampling/sampling_logp_difference/mean": 0.02590002156794071, "step": 1260, "step_time": 11.794867588486522 }, { "clip_ratio/high_max": 0.002203794554225169, "clip_ratio/high_mean": 0.002203794554225169, "clip_ratio/low_mean": 0.002913662715582177, "clip_ratio/low_min": 0.002913662715582177, "clip_ratio/region_mean": 0.005117457269807346, "completions/clipped_ratio": 0.175, "completions/max_length": 1793.2, "completions/max_terminated_length": 1531.7, "completions/mean_length": 1196.125, "completions/mean_terminated_length": 1038.6012084960937, "completions/min_length": 698.6, "completions/min_terminated_length": 698.6, "entropy": 0.2169684995431453, "epoch": 1.8814814814814815, "frac_reward_zero_std": 0.4, "grad_norm": 0.2469620555639267, "learning_rate": 4.111049003750292e-06, "loss": 0.00821465477347374, "num_tokens": 15401800.0, "reward": 0.5274306498467922, "reward_std": 0.43416290134191515, "rewards/correctness_reward/mean": 0.43999999910593035, "rewards/correctness_reward/std": 0.4066426634788513, "rewards/format_reward/mean": 0.08250000327825546, "rewards/format_reward/std": 0.029461687430739403, "rewards/length_efficiency_reward/mean": 0.004930625041015446, "rewards/length_efficiency_reward/std": 0.006214792956598103, "sampling/importance_sampling_ratio/max": 0.868255265057087, "sampling/importance_sampling_ratio/mean": 0.17142380140721797, "sampling/importance_sampling_ratio/min": 0.00047245173363990033, "sampling/sampling_logp_difference/max": 1.9965605735778809, "sampling/sampling_logp_difference/mean": 0.025656230933964252, "step": 1270, "step_time": 13.230381777044386 }, { "clip_ratio/high_max": 0.0028714265616144987, "clip_ratio/high_mean": 0.0028714265616144987, "clip_ratio/low_mean": 0.004278459609486163, "clip_ratio/low_min": 0.004278459609486163, "clip_ratio/region_mean": 0.007149886171100661, "completions/clipped_ratio": 0.2125, "completions/max_length": 1673.9, "completions/max_terminated_length": 1333.8, "completions/mean_length": 1170.6625, "completions/mean_terminated_length": 928.51083984375, "completions/min_length": 571.5, "completions/min_terminated_length": 571.5, "entropy": 0.20816256226971747, "epoch": 1.8962962962962964, "frac_reward_zero_std": 0.3, "grad_norm": 0.0, "learning_rate": 4.0953840192577135e-06, "loss": 0.009239747375249862, "num_tokens": 15507237.0, "reward": 0.6909719112329185, "reward_std": 0.31234672842547295, "rewards/correctness_reward/mean": 0.6074999999254942, "rewards/correctness_reward/std": 0.28452699482440946, "rewards/format_reward/mean": 0.0787500012665987, "rewards/format_reward/std": 0.025840268656611442, "rewards/length_efficiency_reward/mean": 0.004721874947426841, "rewards/length_efficiency_reward/std": 0.005778576957527548, "sampling/importance_sampling_ratio/max": 0.763514044880867, "sampling/importance_sampling_ratio/mean": 0.17392795737832784, "sampling/importance_sampling_ratio/min": 0.0021853368164101994, "sampling/sampling_logp_difference/max": 2.1442254781723022, "sampling/sampling_logp_difference/mean": 0.025523544847965242, "step": 1280, "step_time": 12.46662655491382 }, { "clip_ratio/high_max": 0.003017811046447605, "clip_ratio/high_mean": 0.003017811046447605, "clip_ratio/low_mean": 0.0030181337235262616, "clip_ratio/low_min": 0.0030181337235262616, "clip_ratio/region_mean": 0.0060359447699738665, "completions/clipped_ratio": 0.2, "completions/max_length": 1633.5, "completions/max_terminated_length": 1212.7, "completions/mean_length": 1121.3875, "completions/mean_terminated_length": 867.3317932128906, "completions/min_length": 601.9, "completions/min_terminated_length": 601.9, "entropy": 0.20295926174148918, "epoch": 1.911111111111111, "frac_reward_zero_std": 0.35, "grad_norm": 0.03501857817173004, "learning_rate": 4.079612640505527e-06, "loss": 0.027299195528030396, "num_tokens": 15607224.0, "reward": 0.720631281286478, "reward_std": 0.36971242455765607, "rewards/correctness_reward/mean": 0.635000005364418, "rewards/correctness_reward/std": 0.3410892218351364, "rewards/format_reward/mean": 0.08000000156462192, "rewards/format_reward/std": 0.02474670223891735, "rewards/length_efficiency_reward/mean": 0.005631249980069697, "rewards/length_efficiency_reward/std": 0.005894155381247402, "sampling/importance_sampling_ratio/max": 0.6235600553452969, "sampling/importance_sampling_ratio/mean": 0.12889182101935148, "sampling/importance_sampling_ratio/min": 0.000546511819970874, "sampling/sampling_logp_difference/max": 1.8366664052009583, "sampling/sampling_logp_difference/mean": 0.024739576503634453, "step": 1290, "step_time": 12.069967528339475 }, { "clip_ratio/high_max": 0.0034915779804578053, "clip_ratio/high_mean": 0.0034915779804578053, "clip_ratio/low_mean": 0.004245528468163684, "clip_ratio/low_min": 0.004245528468163684, "clip_ratio/region_mean": 0.0077371064486214895, "completions/clipped_ratio": 0.075, "completions/max_length": 1487.1, "completions/max_terminated_length": 1331.2, "completions/mean_length": 976.7375, "completions/mean_terminated_length": 905.5892944335938, "completions/min_length": 565.9, "completions/min_terminated_length": 565.9, "entropy": 0.22644903073087336, "epoch": 1.925925925925926, "frac_reward_zero_std": 0.3, "grad_norm": 0.011391811072826385, "learning_rate": 4.063735919268194e-06, "loss": -0.017327819764614106, "num_tokens": 15696279.0, "reward": 0.7321062684059143, "reward_std": 0.45831076451577246, "rewards/correctness_reward/mean": 0.6350000023841857, "rewards/correctness_reward/std": 0.44420733451843264, "rewards/format_reward/mean": 0.09250000230967999, "rewards/format_reward/std": 0.01241629347205162, "rewards/length_efficiency_reward/mean": 0.004606249969219789, "rewards/length_efficiency_reward/std": 0.006593531602993608, "sampling/importance_sampling_ratio/max": 0.8096863135695458, "sampling/importance_sampling_ratio/mean": 0.16120916604995728, "sampling/importance_sampling_ratio/min": 0.00044676083607555486, "sampling/sampling_logp_difference/max": 2.589146304130554, "sampling/sampling_logp_difference/mean": 0.029145584627985954, "step": 1300, "step_time": 11.240906596370042 }, { "clip_ratio/high_max": 0.002519258443498984, "clip_ratio/high_mean": 0.002519258443498984, "clip_ratio/low_mean": 0.0040501039024093185, "clip_ratio/low_min": 0.0040501039024093185, "clip_ratio/region_mean": 0.0065693623459083025, "completions/clipped_ratio": 0.1875, "completions/max_length": 1752.7, "completions/max_terminated_length": 1272.6, "completions/mean_length": 1098.1125, "completions/mean_terminated_length": 827.48095703125, "completions/min_length": 560.5, "completions/min_terminated_length": 560.5, "entropy": 0.22003171499818563, "epoch": 1.9407407407407407, "frac_reward_zero_std": 0.25, "grad_norm": 0.17426900565624237, "learning_rate": 4.047754914345338e-06, "loss": 0.010906077921390533, "num_tokens": 15796580.0, "reward": 0.6984106451272964, "reward_std": 0.5082601678557694, "rewards/correctness_reward/mean": 0.6125, "rewards/correctness_reward/std": 0.48209784626960756, "rewards/format_reward/mean": 0.08125000186264515, "rewards/format_reward/std": 0.024200310558080675, "rewards/length_efficiency_reward/mean": 0.004660625015822007, "rewards/length_efficiency_reward/std": 0.005816227343166247, "sampling/importance_sampling_ratio/max": 0.9715241715312004, "sampling/importance_sampling_ratio/mean": 0.1866447314620018, "sampling/importance_sampling_ratio/min": 0.0016717872837944015, "sampling/sampling_logp_difference/max": 2.639928460121155, "sampling/sampling_logp_difference/mean": 0.02584614995867014, "step": 1310, "step_time": 12.732640840951353 }, { "clip_ratio/high_max": 0.0025725374958710744, "clip_ratio/high_mean": 0.0025725374958710744, "clip_ratio/low_mean": 0.003047565426095389, "clip_ratio/low_min": 0.003047565426095389, "clip_ratio/region_mean": 0.005620102921966463, "completions/clipped_ratio": 0.1375, "completions/max_length": 1658.7, "completions/max_terminated_length": 1418.4, "completions/mean_length": 1098.65, "completions/mean_terminated_length": 942.0202453613281, "completions/min_length": 613.1, "completions/min_terminated_length": 613.1, "entropy": 0.22013968797400593, "epoch": 1.9555555555555557, "frac_reward_zero_std": 0.45, "grad_norm": 0.044436778873205185, "learning_rate": 4.031670691491138e-06, "loss": -0.0024984220042824747, "num_tokens": 15896612.0, "reward": 0.6009100090712309, "reward_std": 0.3579779260326177, "rewards/correctness_reward/mean": 0.509999999217689, "rewards/correctness_reward/std": 0.33689771220088005, "rewards/format_reward/mean": 0.08625000193715096, "rewards/format_reward/std": 0.018855085223913194, "rewards/length_efficiency_reward/mean": 0.004660000000149012, "rewards/length_efficiency_reward/std": 0.004605096392333507, "sampling/importance_sampling_ratio/max": 0.37692959159612655, "sampling/importance_sampling_ratio/mean": 0.0862418245524168, "sampling/importance_sampling_ratio/min": 0.0004345066506175499, "sampling/sampling_logp_difference/max": 2.4713928580284117, "sampling/sampling_logp_difference/mean": 0.027397588454186918, "step": 1320, "step_time": 12.224303267151118 }, { "clip_ratio/high_max": 0.002300082556030247, "clip_ratio/high_mean": 0.002300082556030247, "clip_ratio/low_mean": 0.0031073984660906716, "clip_ratio/low_min": 0.0031073984660906716, "clip_ratio/region_mean": 0.005407481022120919, "completions/clipped_ratio": 0.1375, "completions/max_length": 1468.1, "completions/max_terminated_length": 1222.6, "completions/mean_length": 1040.2125, "completions/mean_terminated_length": 896.7756164550781, "completions/min_length": 640.4, "completions/min_terminated_length": 640.4, "entropy": 0.21010440131649374, "epoch": 1.9703703703703703, "frac_reward_zero_std": 0.35, "grad_norm": 0.027566611766815186, "learning_rate": 4.015484323343256e-06, "loss": -0.024973441660404206, "num_tokens": 15992301.0, "reward": 0.8019000321626664, "reward_std": 0.3873798295855522, "rewards/correctness_reward/mean": 0.710000005364418, "rewards/correctness_reward/std": 0.3642969191074371, "rewards/format_reward/mean": 0.08625000230967998, "rewards/format_reward/std": 0.01759178563952446, "rewards/length_efficiency_reward/mean": 0.005650000076275319, "rewards/length_efficiency_reward/std": 0.008262032037600875, "sampling/importance_sampling_ratio/max": 0.7472231671214103, "sampling/importance_sampling_ratio/mean": 0.1453169144690037, "sampling/importance_sampling_ratio/min": 0.0006308795884251595, "sampling/sampling_logp_difference/max": 2.4173866629600527, "sampling/sampling_logp_difference/mean": 0.026543315313756467, "step": 1330, "step_time": 11.344096638727933 }, { "clip_ratio/high_max": 0.0027879546672920697, "clip_ratio/high_mean": 0.0027879546672920697, "clip_ratio/low_mean": 0.003714343812316656, "clip_ratio/low_min": 0.003714343812316656, "clip_ratio/region_mean": 0.006502298479608726, "completions/clipped_ratio": 0.25, "completions/max_length": 1975.1, "completions/max_terminated_length": 1609.9, "completions/mean_length": 1348.5, "completions/mean_terminated_length": 1132.1916809082031, "completions/min_length": 774.4, "completions/min_terminated_length": 774.4, "entropy": 0.2099521217867732, "epoch": 1.9851851851851852, "frac_reward_zero_std": 0.3, "grad_norm": 0.0, "learning_rate": 3.9991968893513e-06, "loss": 0.02436164766550064, "num_tokens": 16113793.0, "reward": 0.6183787727728486, "reward_std": 0.46564867021515965, "rewards/correctness_reward/mean": 0.5375000048428774, "rewards/correctness_reward/std": 0.4319961749017239, "rewards/format_reward/mean": 0.0750000024214387, "rewards/format_reward/std": 0.031648820266127586, "rewards/length_efficiency_reward/mean": 0.0058787499088794, "rewards/length_efficiency_reward/std": 0.008427160326391459, "sampling/importance_sampling_ratio/max": 1.2571668196469545, "sampling/importance_sampling_ratio/mean": 0.2263078280724585, "sampling/importance_sampling_ratio/min": 0.0008584225031881942, "sampling/sampling_logp_difference/max": 1.9222394585609437, "sampling/sampling_logp_difference/mean": 0.022203636541962623, "step": 1340, "step_time": 14.191794874053448 }, { "clip_ratio/high_max": 0.0025954135198844597, "clip_ratio/high_mean": 0.0025954135198844597, "clip_ratio/low_mean": 0.004388632089830935, "clip_ratio/low_min": 0.004388632089830935, "clip_ratio/region_mean": 0.006984045609715395, "completions/clipped_ratio": 0.1, "completions/max_length": 1515.8, "completions/max_terminated_length": 1273.4, "completions/mean_length": 980.3875, "completions/mean_terminated_length": 869.945849609375, "completions/min_length": 536.5, "completions/min_terminated_length": 536.5, "entropy": 0.23564114002510905, "epoch": 2.0, "frac_reward_zero_std": 0.3, "grad_norm": 0.04044508561491966, "learning_rate": 3.982809475704839e-06, "loss": 0.05708513259887695, "num_tokens": 16204300.0, "reward": 0.7384356550872326, "reward_std": 0.22342307888902724, "rewards/correctness_reward/mean": 0.6424999989569187, "rewards/correctness_reward/std": 0.2050539344549179, "rewards/format_reward/mean": 0.0900000024586916, "rewards/format_reward/std": 0.01704539395868778, "rewards/length_efficiency_reward/mean": 0.005935625021811575, "rewards/length_efficiency_reward/std": 0.006445764622185379, "sampling/importance_sampling_ratio/max": 0.8739755980670452, "sampling/importance_sampling_ratio/mean": 0.178867843747139, "sampling/importance_sampling_ratio/min": 0.012254255647349055, "sampling/sampling_logp_difference/max": 1.9040767550468445, "sampling/sampling_logp_difference/mean": 0.02676877025514841, "step": 1350, "step_time": 11.51403228668496 }, { "epoch": 2.0, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.10166666666666667, "eval_completions/max_length": 1576.9733333333334, "eval_completions/max_terminated_length": 1432.8666666666666, "eval_completions/mean_length": 1074.2733333333333, "eval_completions/mean_terminated_length": 972.7034822591146, "eval_completions/min_length": 620.8266666666667, "eval_completions/min_terminated_length": 620.8266666666667, "eval_entropy": 0.23130131969849269, "eval_frac_reward_zero_std": 0.2866666666666667, "eval_loss": -0.009707366116344929, "eval_num_tokens": 16204300.0, "eval_reward": 0.698700684060653, "eval_reward_std": 0.36404518647585066, "eval_rewards/correctness_reward/mean": 0.6030000017086665, "eval_rewards/correctness_reward/std": 0.3474981613953908, "eval_rewards/format_reward/mean": 0.08983333524316549, "eval_rewards/format_reward/std": 0.013793199410041174, "eval_rewards/length_efficiency_reward/mean": 0.00586733336347, "eval_rewards/length_efficiency_reward/std": 0.006015713486510019, "eval_runtime": 685.8583, "eval_samples_per_second": 0.219, "eval_sampling/importance_sampling_ratio/max": 0.952868836025397, "eval_sampling/importance_sampling_ratio/mean": 0.21751872879763445, "eval_sampling/importance_sampling_ratio/min": 0.0024113683705945734, "eval_sampling/sampling_logp_difference/max": 2.0751589361826577, "eval_sampling/sampling_logp_difference/mean": 0.024513648400704066, "eval_steps_per_second": 0.028, "step": 1350 } ], "logging_steps": 10, "max_steps": 4050, "num_input_tokens_seen": 16204300, "num_train_epochs": 6, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }