{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.4076086956521739, "eval_steps": 500, "global_step": 150, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1671.2799072265625, "completions/mean_terminated_length": 1398.4827880859375, "completions/min_length": 992.0, "completions/min_terminated_length": 992.0, "entropy": 0.21248915791511536, "epoch": 0.002717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.01607886701822281, "learning_rate": 0.0, "loss": 0.07359933853149414, "num_tokens": 86304.0, "reward": -0.21605467796325684, "reward_std": 0.656523585319519, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.8160547018051147, "rewards/length_penalty/std": 0.18964654207229614, "sampling/importance_sampling_ratio/max": 1.5222556591033936, "sampling/importance_sampling_ratio/mean": 0.9927070140838623, "sampling/importance_sampling_ratio/min": 0.6435883045196533, "sampling/sampling_logp_difference/max": 0.44069600105285645, "sampling/sampling_logp_difference/mean": 0.013988969847559929, "step": 1, "step_time": 27.394495805958286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6800000071525574, "completions/max_length": 2048.0, "completions/max_terminated_length": 1991.0, "completions/mean_length": 1941.47998046875, "completions/mean_terminated_length": 1715.125, "completions/min_length": 1079.0, "completions/min_terminated_length": 1079.0, "entropy": 0.2905632793903351, "epoch": 0.005434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.01730405166745186, "learning_rate": 5e-06, "loss": 0.07159668952226639, "num_tokens": 186618.0, "reward": -0.7279882431030273, "reward_std": 0.4917677640914917, "rewards/correctness/mean": 0.2199999988079071, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.9479882717132568, "rewards/length_penalty/std": 0.10518474876880646, "sampling/importance_sampling_ratio/max": 1.450933814048767, "sampling/importance_sampling_ratio/mean": 0.9901063442230225, "sampling/importance_sampling_ratio/min": 0.3688630163669586, "sampling/sampling_logp_difference/max": 0.9973299503326416, "sampling/sampling_logp_difference/mean": 0.01780563034117222, "step": 2, "step_time": 28.991049223113805 }, { "clip_ratio/high_max": 0.0009276511613279581, "clip_ratio/high_mean": 0.0009276511613279581, "clip_ratio/low_mean": 0.00012337134103290737, "clip_ratio/low_min": 0.00012337134103290737, "clip_ratio/region_mean": 0.0010510225081816315, "completions/clipped_ratio": 0.3999999761581421, "completions/max_length": 2048.0, "completions/max_terminated_length": 2048.0, "completions/mean_length": 1714.760009765625, "completions/mean_terminated_length": 1492.60009765625, "completions/min_length": 753.0, "completions/min_terminated_length": 753.0, "entropy": 0.2737257957458496, "epoch": 0.008152173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.017563248053193092, "learning_rate": 1e-05, "loss": 0.09476927667856216, "num_tokens": 274626.0, "reward": -0.21728515625, "reward_std": 0.6381246447563171, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143346309662, "rewards/length_penalty/mean": -0.8372851610183716, "rewards/length_penalty/std": 0.20151396095752716, "sampling/importance_sampling_ratio/max": 1.426476001739502, "sampling/importance_sampling_ratio/mean": 0.9904518723487854, "sampling/importance_sampling_ratio/min": 0.6474350690841675, "sampling/sampling_logp_difference/max": 0.4347367286682129, "sampling/sampling_logp_difference/mean": 0.01698637194931507, "step": 3, "step_time": 27.53572188084945 }, { "clip_ratio/high_max": 0.0008016214531380683, "clip_ratio/high_mean": 0.0008016214531380683, "clip_ratio/low_mean": 0.00010628659802023322, "clip_ratio/low_min": 0.00010628659802023322, "clip_ratio/region_mean": 0.0009079080424271524, "completions/clipped_ratio": 0.3199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 1991.0, "completions/mean_length": 1506.5399169921875, "completions/mean_terminated_length": 1251.7353515625, "completions/min_length": 638.0, "completions/min_terminated_length": 638.0, "entropy": 0.25913241803646087, "epoch": 0.010869565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.017625654116272926, "learning_rate": 1.5e-05, "loss": 0.12047722935676575, "num_tokens": 352853.0, "reward": -0.0356152318418026, "reward_std": 0.6583986878395081, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.7356152534484863, "rewards/length_penalty/std": 0.23944713175296783, "sampling/importance_sampling_ratio/max": 1.4886895418167114, "sampling/importance_sampling_ratio/mean": 0.9911117553710938, "sampling/importance_sampling_ratio/min": 0.6022539734840393, "sampling/sampling_logp_difference/max": 0.5070760250091553, "sampling/sampling_logp_difference/mean": 0.016714798286557198, "step": 4, "step_time": 27.26718003093265 }, { "clip_ratio/high_max": 0.00031515124428551646, "clip_ratio/high_mean": 0.00031515124428551646, "clip_ratio/low_mean": 0.0001696181825536769, "clip_ratio/low_min": 0.0001696181825536769, "clip_ratio/region_mean": 0.00048476942392881026, "completions/clipped_ratio": 0.7599999904632568, "completions/max_length": 2048.0, "completions/max_terminated_length": 2022.0, "completions/mean_length": 1914.3599853515625, "completions/mean_terminated_length": 1491.166748046875, "completions/min_length": 1072.0, "completions/min_terminated_length": 1072.0, "entropy": 0.2717311054468155, "epoch": 0.01358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.018083127215504646, "learning_rate": 2e-05, "loss": 0.08188383281230927, "num_tokens": 452281.0, "reward": -0.6947460770606995, "reward_std": 0.5523213148117065, "rewards/correctness/mean": 0.23999999463558197, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.9347460865974426, "rewards/length_penalty/std": 0.13313990831375122, "sampling/importance_sampling_ratio/max": 1.7996419668197632, "sampling/importance_sampling_ratio/mean": 0.9906188249588013, "sampling/importance_sampling_ratio/min": 0.6196915507316589, "sampling/sampling_logp_difference/max": 0.5875877141952515, "sampling/sampling_logp_difference/mean": 0.01734403893351555, "step": 5, "step_time": 28.98220992088318 }, { "clip_ratio/high_max": 0.0012173810973763466, "clip_ratio/high_mean": 0.0012173810973763466, "clip_ratio/low_mean": 8.390831935685128e-05, "clip_ratio/low_min": 8.390831935685128e-05, "clip_ratio/region_mean": 0.0013012894080020488, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 2042.0, "completions/mean_length": 1670.679931640625, "completions/mean_terminated_length": 1397.4482421875, "completions/min_length": 764.0, "completions/min_terminated_length": 764.0, "entropy": 0.25660490691661836, "epoch": 0.016304347826086956, "frac_reward_zero_std": 0.0, "grad_norm": 0.016991887241601944, "learning_rate": 2.5e-05, "loss": 0.07579805701971054, "num_tokens": 538805.0, "reward": -0.23576171696186066, "reward_std": 0.6744439005851746, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.4985693693161011, "rewards/length_penalty/mean": -0.8157617449760437, "rewards/length_penalty/std": 0.21996504068374634, "sampling/importance_sampling_ratio/max": 1.4670546054840088, "sampling/importance_sampling_ratio/mean": 0.9912019371986389, "sampling/importance_sampling_ratio/min": 0.6486796736717224, "sampling/sampling_logp_difference/max": 0.4328162670135498, "sampling/sampling_logp_difference/mean": 0.01650330238044262, "step": 6, "step_time": 27.390100880060345 }, { "clip_ratio/high_max": 0.0008734886534512043, "clip_ratio/high_mean": 0.0008734886534512043, "clip_ratio/low_mean": 6.886005430715159e-05, "clip_ratio/low_min": 6.886005430715159e-05, "clip_ratio/region_mean": 0.0009423487004823983, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 1985.0, "completions/mean_length": 1602.5799560546875, "completions/mean_terminated_length": 1411.6856689453125, "completions/min_length": 862.0, "completions/min_terminated_length": 862.0, "entropy": 0.24657512605190277, "epoch": 0.019021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.018574461340904236, "learning_rate": 3e-05, "loss": 0.10817548632621765, "num_tokens": 621164.0, "reward": -0.08250976353883743, "reward_std": 0.6237230896949768, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.7825097441673279, "rewards/length_penalty/std": 0.2039300501346588, "sampling/importance_sampling_ratio/max": 1.899301528930664, "sampling/importance_sampling_ratio/mean": 0.9912979006767273, "sampling/importance_sampling_ratio/min": 0.5936238765716553, "sampling/sampling_logp_difference/max": 0.6414861679077148, "sampling/sampling_logp_difference/mean": 0.015865743160247803, "step": 7, "step_time": 27.102628064341843 }, { "clip_ratio/high_max": 0.00030049889464862646, "clip_ratio/high_mean": 0.00030049889464862646, "clip_ratio/low_mean": 0.0001223457460582722, "clip_ratio/low_min": 0.0001223457460582722, "clip_ratio/region_mean": 0.00042284463997930286, "completions/clipped_ratio": 0.5399999618530273, "completions/max_length": 2048.0, "completions/max_terminated_length": 2021.0, "completions/mean_length": 1799.3199462890625, "completions/mean_terminated_length": 1507.391357421875, "completions/min_length": 766.0, "completions/min_terminated_length": 766.0, "entropy": 0.21842886209487916, "epoch": 0.021739130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.01700979843735695, "learning_rate": 3.5e-05, "loss": 0.07161665707826614, "num_tokens": 713280.0, "reward": -0.4185742139816284, "reward_std": 0.6441221833229065, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.8785741925239563, "rewards/length_penalty/std": 0.16615615785121918, "sampling/importance_sampling_ratio/max": 1.6265782117843628, "sampling/importance_sampling_ratio/mean": 0.9922990202903748, "sampling/importance_sampling_ratio/min": 0.634637713432312, "sampling/sampling_logp_difference/max": 0.4864785671234131, "sampling/sampling_logp_difference/mean": 0.014363318681716919, "step": 8, "step_time": 28.30564660113305 }, { "clip_ratio/high_max": 0.0009418363450095057, "clip_ratio/high_mean": 0.0009418363450095057, "clip_ratio/low_mean": 9.628144616726785e-05, "clip_ratio/low_min": 9.628144616726785e-05, "clip_ratio/region_mean": 0.0010381177882663906, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 2048.0, "completions/mean_length": 1622.97998046875, "completions/mean_terminated_length": 1503.1025390625, "completions/min_length": 873.0, "completions/min_terminated_length": 873.0, "entropy": 0.22798832952976228, "epoch": 0.024456521739130436, "frac_reward_zero_std": 0.0, "grad_norm": 0.018526818603277206, "learning_rate": 4e-05, "loss": 0.11752811819314957, "num_tokens": 796899.0, "reward": -0.1524706929922104, "reward_std": 0.5888569355010986, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732054233551, "rewards/length_penalty/mean": -0.7924706935882568, "rewards/length_penalty/std": 0.19514665007591248, "sampling/importance_sampling_ratio/max": 1.4198806285858154, "sampling/importance_sampling_ratio/mean": 0.9919716119766235, "sampling/importance_sampling_ratio/min": 0.511864960193634, "sampling/sampling_logp_difference/max": 0.6696943640708923, "sampling/sampling_logp_difference/mean": 0.015113740228116512, "step": 9, "step_time": 27.674709191778675 }, { "clip_ratio/high_max": 0.0008836857683490962, "clip_ratio/high_mean": 0.0008836857683490962, "clip_ratio/low_mean": 0.0001610520586837083, "clip_ratio/low_min": 0.0001610520586837083, "clip_ratio/region_mean": 0.0010447378270328044, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 1945.0, "completions/mean_length": 1527.5599365234375, "completions/mean_terminated_length": 1325.1666259765625, "completions/min_length": 650.0, "completions/min_terminated_length": 650.0, "entropy": 0.24462299942970275, "epoch": 0.02717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.017739074304699898, "learning_rate": 4.5e-05, "loss": 0.11460381746292114, "num_tokens": 878097.0, "reward": -0.0858789011836052, "reward_std": 0.651462733745575, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.47851812839508057, "rewards/length_penalty/mean": -0.7458789348602295, "rewards/length_penalty/std": 0.22174151241779327, "sampling/importance_sampling_ratio/max": 1.4572726488113403, "sampling/importance_sampling_ratio/mean": 0.9913933277130127, "sampling/importance_sampling_ratio/min": 0.5709614157676697, "sampling/sampling_logp_difference/max": 0.5604336261749268, "sampling/sampling_logp_difference/mean": 0.01590959168970585, "step": 10, "step_time": 27.32456138706766 }, { "clip_ratio/high_max": 0.0005791342351585627, "clip_ratio/high_mean": 0.0005791342351585627, "clip_ratio/low_mean": 0.00020766998059116304, "clip_ratio/low_min": 0.00020766998059116304, "clip_ratio/region_mean": 0.0007868041866458952, "completions/clipped_ratio": 0.5799999833106995, "completions/max_length": 2048.0, "completions/max_terminated_length": 2005.0, "completions/mean_length": 1906.219970703125, "completions/mean_terminated_length": 1710.4285888671875, "completions/min_length": 1323.0, "completions/min_terminated_length": 1323.0, "entropy": 0.2523061394691467, "epoch": 0.029891304347826088, "frac_reward_zero_std": 0.0, "grad_norm": 0.017356909811496735, "learning_rate": 5e-05, "loss": 0.074811190366745, "num_tokens": 975828.0, "reward": -0.4507714807987213, "reward_std": 0.5823943614959717, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.9307714700698853, "rewards/length_penalty/std": 0.1049875020980835, "sampling/importance_sampling_ratio/max": 1.9958995580673218, "sampling/importance_sampling_ratio/mean": 0.9913272857666016, "sampling/importance_sampling_ratio/min": 0.6561847925186157, "sampling/sampling_logp_difference/max": 0.6910948753356934, "sampling/sampling_logp_difference/mean": 0.015796277672052383, "step": 11, "step_time": 28.954787808004767 }, { "clip_ratio/high_max": 0.0004949859721818939, "clip_ratio/high_mean": 0.0004949859721818939, "clip_ratio/low_mean": 0.0001681178982835263, "clip_ratio/low_min": 0.0001681178982835263, "clip_ratio/region_mean": 0.000663103861734271, "completions/clipped_ratio": 0.4599999785423279, "completions/max_length": 2048.0, "completions/max_terminated_length": 1984.0, "completions/mean_length": 1742.43994140625, "completions/mean_terminated_length": 1482.148193359375, "completions/min_length": 974.0, "completions/min_terminated_length": 974.0, "entropy": 0.2790819376707077, "epoch": 0.03260869565217391, "frac_reward_zero_std": 0.0, "grad_norm": 0.017921727150678635, "learning_rate": 5e-05, "loss": 0.12824325263500214, "num_tokens": 1065590.0, "reward": -0.3508007824420929, "reward_std": 0.6496888995170593, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.8508007526397705, "rewards/length_penalty/std": 0.1730204075574875, "sampling/importance_sampling_ratio/max": 1.4403636455535889, "sampling/importance_sampling_ratio/mean": 0.9905098676681519, "sampling/importance_sampling_ratio/min": 0.5996456146240234, "sampling/sampling_logp_difference/max": 0.5114164352416992, "sampling/sampling_logp_difference/mean": 0.01758064143359661, "step": 12, "step_time": 27.76311965705827 }, { "clip_ratio/high_max": 0.0006719783181324601, "clip_ratio/high_mean": 0.0006719783181324601, "clip_ratio/low_mean": 0.00014902167167747394, "clip_ratio/low_min": 0.00014902167167747394, "clip_ratio/region_mean": 0.0008209999883547425, "completions/clipped_ratio": 0.4399999976158142, "completions/max_length": 2048.0, "completions/max_terminated_length": 2033.0, "completions/mean_length": 1722.1400146484375, "completions/mean_terminated_length": 1466.107177734375, "completions/min_length": 936.0, "completions/min_terminated_length": 936.0, "entropy": 0.26495963633060454, "epoch": 0.035326086956521736, "frac_reward_zero_std": 0.0, "grad_norm": 0.01803506910800934, "learning_rate": 5e-05, "loss": 0.10340610891580582, "num_tokens": 1154717.0, "reward": -0.3808886706829071, "reward_std": 0.6363792419433594, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.8408886790275574, "rewards/length_penalty/std": 0.17392964661121368, "sampling/importance_sampling_ratio/max": 1.6443123817443848, "sampling/importance_sampling_ratio/mean": 0.9907426238059998, "sampling/importance_sampling_ratio/min": 0.6476600170135498, "sampling/sampling_logp_difference/max": 0.49732232093811035, "sampling/sampling_logp_difference/mean": 0.01705038733780384, "step": 13, "step_time": 27.700532860821113 }, { "clip_ratio/high_max": 0.000790283753303811, "clip_ratio/high_mean": 0.000790283753303811, "clip_ratio/low_mean": 0.00015664642269257455, "clip_ratio/low_min": 0.00015664642269257455, "clip_ratio/region_mean": 0.0009469301789067685, "completions/clipped_ratio": 0.4399999976158142, "completions/max_length": 2048.0, "completions/max_terminated_length": 1947.0, "completions/mean_length": 1707.39990234375, "completions/mean_terminated_length": 1439.7857666015625, "completions/min_length": 939.0, "completions/min_terminated_length": 939.0, "entropy": 0.25884707272052765, "epoch": 0.03804347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.01831369660794735, "learning_rate": 5e-05, "loss": 0.10717891156673431, "num_tokens": 1242647.0, "reward": -0.2536914050579071, "reward_std": 0.6496968865394592, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.4985693693161011, "rewards/length_penalty/mean": -0.833691418170929, "rewards/length_penalty/std": 0.17602378129959106, "sampling/importance_sampling_ratio/max": 1.4137200117111206, "sampling/importance_sampling_ratio/mean": 0.9911543130874634, "sampling/importance_sampling_ratio/min": 0.6645229458808899, "sampling/sampling_logp_difference/max": 0.40868592262268066, "sampling/sampling_logp_difference/mean": 0.016555489972233772, "step": 14, "step_time": 28.091995016904548 }, { "clip_ratio/high_max": 0.0007948186947032809, "clip_ratio/high_mean": 0.0007948186947032809, "clip_ratio/low_mean": 6.400095226126723e-05, "clip_ratio/low_min": 6.400095226126723e-05, "clip_ratio/region_mean": 0.0008588196593336761, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 2024.0, "completions/mean_length": 1566.659912109375, "completions/mean_terminated_length": 1360.3714599609375, "completions/min_length": 923.0, "completions/min_terminated_length": 923.0, "entropy": 0.19630298018455505, "epoch": 0.04076086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.01908237673342228, "learning_rate": 5e-05, "loss": 0.10991621017456055, "num_tokens": 1323560.0, "reward": -0.2849707007408142, "reward_std": 0.6005774140357971, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.7649707198143005, "rewards/length_penalty/std": 0.1954003870487213, "sampling/importance_sampling_ratio/max": 1.4597245454788208, "sampling/importance_sampling_ratio/mean": 0.9932765960693359, "sampling/importance_sampling_ratio/min": 0.48890236020088196, "sampling/sampling_logp_difference/max": 0.7155925035476685, "sampling/sampling_logp_difference/mean": 0.013154168613255024, "step": 15, "step_time": 27.622011815896258 }, { "clip_ratio/high_max": 0.0007680640854232478, "clip_ratio/high_mean": 0.0007680640854232478, "clip_ratio/low_mean": 0.00010441521517350339, "clip_ratio/low_min": 0.00010441521517350339, "clip_ratio/region_mean": 0.0008724792875000276, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 2038.0, "completions/mean_length": 1529.47998046875, "completions/mean_terminated_length": 1327.8333740234375, "completions/min_length": 648.0, "completions/min_terminated_length": 648.0, "entropy": 0.2527110666036606, "epoch": 0.043478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.019366132095456123, "learning_rate": 5e-05, "loss": 0.1180194765329361, "num_tokens": 1404134.0, "reward": -0.02681640535593033, "reward_std": 0.6313645839691162, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.7468163967132568, "rewards/length_penalty/std": 0.21961458027362823, "sampling/importance_sampling_ratio/max": 1.4531008005142212, "sampling/importance_sampling_ratio/mean": 0.990881621837616, "sampling/importance_sampling_ratio/min": 0.45519256591796875, "sampling/sampling_logp_difference/max": 0.7870347499847412, "sampling/sampling_logp_difference/mean": 0.016263339668512344, "step": 16, "step_time": 26.84932530601509 }, { "clip_ratio/high_max": 0.0012978371232748032, "clip_ratio/high_mean": 0.0012978371232748032, "clip_ratio/low_mean": 4.327131027821451e-05, "clip_ratio/low_min": 4.327131027821451e-05, "clip_ratio/region_mean": 0.001341108442284167, "completions/clipped_ratio": 0.5600000023841858, "completions/max_length": 2048.0, "completions/max_terminated_length": 2007.0, "completions/mean_length": 1768.919921875, "completions/mean_terminated_length": 1413.727294921875, "completions/min_length": 865.0, "completions/min_terminated_length": 865.0, "entropy": 0.23181203305721282, "epoch": 0.04619565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.016713930293917656, "learning_rate": 5e-05, "loss": 0.08152732998132706, "num_tokens": 1495230.0, "reward": -0.4237304627895355, "reward_std": 0.6704393029212952, "rewards/correctness/mean": 0.4399999976158142, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.8637304902076721, "rewards/length_penalty/std": 0.20572726428508759, "sampling/importance_sampling_ratio/max": 1.561582088470459, "sampling/importance_sampling_ratio/mean": 0.9919421076774597, "sampling/importance_sampling_ratio/min": 0.543276309967041, "sampling/sampling_logp_difference/max": 0.6101372241973877, "sampling/sampling_logp_difference/mean": 0.015333480201661587, "step": 17, "step_time": 27.873049326473847 }, { "clip_ratio/high_max": 0.0006598436157219112, "clip_ratio/high_mean": 0.0006598436157219112, "clip_ratio/low_mean": 0.00010337726125726476, "clip_ratio/low_min": 0.00010337726125726476, "clip_ratio/region_mean": 0.0007632208871655166, "completions/clipped_ratio": 0.35999998450279236, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1746.0799560546875, "completions/mean_terminated_length": 1576.25, "completions/min_length": 1028.0, "completions/min_terminated_length": 1028.0, "entropy": 0.21664705574512483, "epoch": 0.04891304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.018009476363658905, "learning_rate": 5e-05, "loss": 0.08006907999515533, "num_tokens": 1585034.0, "reward": -0.2725781202316284, "reward_std": 0.625697135925293, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.8525781035423279, "rewards/length_penalty/std": 0.15941378474235535, "sampling/importance_sampling_ratio/max": 1.639607310295105, "sampling/importance_sampling_ratio/mean": 0.9925068020820618, "sampling/importance_sampling_ratio/min": 0.4967416226863861, "sampling/sampling_logp_difference/max": 0.6996852159500122, "sampling/sampling_logp_difference/mean": 0.013985401019454002, "step": 18, "step_time": 28.311645316891372 }, { "clip_ratio/high_max": 0.000741867849137634, "clip_ratio/high_mean": 0.000741867849137634, "clip_ratio/low_mean": 0.00018152535776607692, "clip_ratio/low_min": 0.00018152535776607692, "clip_ratio/region_mean": 0.0009233932243660093, "completions/clipped_ratio": 0.35999998450279236, "completions/max_length": 2048.0, "completions/max_terminated_length": 2011.0, "completions/mean_length": 1686.260009765625, "completions/mean_terminated_length": 1482.78125, "completions/min_length": 740.0, "completions/min_terminated_length": 740.0, "entropy": 0.2879431128501892, "epoch": 0.051630434782608696, "frac_reward_zero_std": 0.0, "grad_norm": 0.019337492063641548, "learning_rate": 5e-05, "loss": 0.08359130471944809, "num_tokens": 1672777.0, "reward": -0.18336912989616394, "reward_std": 0.6349277496337891, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.8233691453933716, "rewards/length_penalty/std": 0.19560563564300537, "sampling/importance_sampling_ratio/max": 1.4635835886001587, "sampling/importance_sampling_ratio/mean": 0.98971027135849, "sampling/importance_sampling_ratio/min": 0.635256290435791, "sampling/sampling_logp_difference/max": 0.45372676849365234, "sampling/sampling_logp_difference/mean": 0.018413560464978218, "step": 19, "step_time": 27.62987573328428 }, { "clip_ratio/high_max": 0.0006679994985461235, "clip_ratio/high_mean": 0.0006679994985461235, "clip_ratio/low_mean": 5.8021536096930505e-05, "clip_ratio/low_min": 5.8021536096930505e-05, "clip_ratio/region_mean": 0.0007260210230015218, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 2046.0, "completions/mean_length": 1787.9599609375, "completions/mean_terminated_length": 1599.6551513671875, "completions/min_length": 775.0, "completions/min_terminated_length": 775.0, "entropy": 0.2634012043476105, "epoch": 0.05434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.018007466569542885, "learning_rate": 5e-05, "loss": 0.10140140354633331, "num_tokens": 1764765.0, "reward": -0.3130273222923279, "reward_std": 0.627247154712677, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.8730273246765137, "rewards/length_penalty/std": 0.17675596475601196, "sampling/importance_sampling_ratio/max": 1.6090019941329956, "sampling/importance_sampling_ratio/mean": 0.9909850358963013, "sampling/importance_sampling_ratio/min": 0.6370789408683777, "sampling/sampling_logp_difference/max": 0.475614070892334, "sampling/sampling_logp_difference/mean": 0.01683841086924076, "step": 20, "step_time": 28.34779040887952 }, { "clip_ratio/high_max": 0.000976240006275475, "clip_ratio/high_mean": 0.000976240006275475, "clip_ratio/low_mean": 7.026160965324379e-05, "clip_ratio/low_min": 7.026160965324379e-05, "clip_ratio/region_mean": 0.00104650161229074, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 2037.0, "completions/mean_length": 1709.5399169921875, "completions/mean_terminated_length": 1564.4857177734375, "completions/min_length": 1061.0, "completions/min_terminated_length": 1061.0, "entropy": 0.20998104512691498, "epoch": 0.057065217391304345, "frac_reward_zero_std": 0.0, "grad_norm": 0.018870066851377487, "learning_rate": 5e-05, "loss": 0.08603541553020477, "num_tokens": 1853652.0, "reward": -0.1347363293170929, "reward_std": 0.5705064535140991, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.8347363471984863, "rewards/length_penalty/std": 0.15160603821277618, "sampling/importance_sampling_ratio/max": 1.4565383195877075, "sampling/importance_sampling_ratio/mean": 0.9924175143241882, "sampling/importance_sampling_ratio/min": 0.6045414805412292, "sampling/sampling_logp_difference/max": 0.5032849311828613, "sampling/sampling_logp_difference/mean": 0.01400977373123169, "step": 21, "step_time": 28.186939923092723 }, { "clip_ratio/high_max": 0.0007286557811312377, "clip_ratio/high_mean": 0.0007286557811312377, "clip_ratio/low_mean": 5.019558957428671e-05, "clip_ratio/low_min": 5.019558957428671e-05, "clip_ratio/region_mean": 0.0007788513612467796, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 2031.0, "completions/mean_length": 1543.7799072265625, "completions/mean_terminated_length": 1327.6856689453125, "completions/min_length": 758.0, "completions/min_terminated_length": 758.0, "entropy": 0.2528064101934433, "epoch": 0.059782608695652176, "frac_reward_zero_std": 0.0, "grad_norm": 0.019696656614542007, "learning_rate": 5e-05, "loss": 0.10337188839912415, "num_tokens": 1935911.0, "reward": -0.05379882827401161, "reward_std": 0.6398226022720337, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.7537988424301147, "rewards/length_penalty/std": 0.21059876680374146, "sampling/importance_sampling_ratio/max": 2.2368545532226562, "sampling/importance_sampling_ratio/mean": 0.9912877082824707, "sampling/importance_sampling_ratio/min": 0.6480669975280762, "sampling/sampling_logp_difference/max": 0.8050706386566162, "sampling/sampling_logp_difference/mean": 0.016178736463189125, "step": 22, "step_time": 27.681587701197714 }, { "clip_ratio/high_max": 0.0008125284919515252, "clip_ratio/high_mean": 0.0008125284919515252, "clip_ratio/low_mean": 0.00017936505464604123, "clip_ratio/low_min": 0.00017936505464604123, "clip_ratio/region_mean": 0.000991893548052758, "completions/clipped_ratio": 0.3400000035762787, "completions/max_length": 2048.0, "completions/max_terminated_length": 2040.0, "completions/mean_length": 1664.02001953125, "completions/mean_terminated_length": 1466.212158203125, "completions/min_length": 824.0, "completions/min_terminated_length": 824.0, "entropy": 0.27761187553405764, "epoch": 0.0625, "frac_reward_zero_std": 0.0, "grad_norm": 0.021949486806988716, "learning_rate": 5e-05, "loss": 0.11840050667524338, "num_tokens": 2025232.0, "reward": -0.2525097727775574, "reward_std": 0.6308436393737793, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.8125097751617432, "rewards/length_penalty/std": 0.18829484283924103, "sampling/importance_sampling_ratio/max": 1.6358195543289185, "sampling/importance_sampling_ratio/mean": 0.9903144836425781, "sampling/importance_sampling_ratio/min": 0.5315372943878174, "sampling/sampling_logp_difference/max": 0.6319819688796997, "sampling/sampling_logp_difference/mean": 0.017899619415402412, "step": 23, "step_time": 28.560938307084143 }, { "clip_ratio/high_max": 0.0005096927285194397, "clip_ratio/high_mean": 0.0005096927285194397, "clip_ratio/low_mean": 8.583837043261156e-05, "clip_ratio/low_min": 8.583837043261156e-05, "clip_ratio/region_mean": 0.0005955311004072428, "completions/clipped_ratio": 0.4399999976158142, "completions/max_length": 2048.0, "completions/max_terminated_length": 2042.0, "completions/mean_length": 1590.5599365234375, "completions/mean_terminated_length": 1231.1429443359375, "completions/min_length": 672.0, "completions/min_terminated_length": 672.0, "entropy": 0.23703448474407196, "epoch": 0.06521739130434782, "frac_reward_zero_std": 0.0, "grad_norm": 0.019747884944081306, "learning_rate": 5e-05, "loss": 0.09910769760608673, "num_tokens": 2107050.0, "reward": -0.2166406214237213, "reward_std": 0.7164656519889832, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.7766406536102295, "rewards/length_penalty/std": 0.24764245748519897, "sampling/importance_sampling_ratio/max": 1.457215428352356, "sampling/importance_sampling_ratio/mean": 0.9916846752166748, "sampling/importance_sampling_ratio/min": 0.5029337406158447, "sampling/sampling_logp_difference/max": 0.6872968673706055, "sampling/sampling_logp_difference/mean": 0.015448657795786858, "step": 24, "step_time": 27.499916886212304 }, { "clip_ratio/high_max": 0.0008482657605782152, "clip_ratio/high_mean": 0.0008482657605782152, "clip_ratio/low_mean": 0.00022012473782524468, "clip_ratio/low_min": 0.00022012473782524468, "clip_ratio/region_mean": 0.0010683904984034598, "completions/clipped_ratio": 0.23999999463558197, "completions/max_length": 2048.0, "completions/max_terminated_length": 1919.0, "completions/mean_length": 1442.97998046875, "completions/mean_terminated_length": 1251.9210205078125, "completions/min_length": 633.0, "completions/min_terminated_length": 633.0, "entropy": 0.2630555659532547, "epoch": 0.06793478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.021064141765236855, "learning_rate": 5e-05, "loss": 0.11863033473491669, "num_tokens": 2181599.0, "reward": 0.055419921875, "reward_std": 0.6141738295555115, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.7045800685882568, "rewards/length_penalty/std": 0.21538712084293365, "sampling/importance_sampling_ratio/max": 1.5541377067565918, "sampling/importance_sampling_ratio/mean": 0.990498960018158, "sampling/importance_sampling_ratio/min": 0.40694311261177063, "sampling/sampling_logp_difference/max": 0.8990819454193115, "sampling/sampling_logp_difference/mean": 0.017718778923153877, "step": 25, "step_time": 26.41467908094637 }, { "clip_ratio/high_max": 0.000819263607263565, "clip_ratio/high_mean": 0.000819263607263565, "clip_ratio/low_mean": 0.00011166772019350901, "clip_ratio/low_min": 0.00011166772019350901, "clip_ratio/region_mean": 0.0009309313609264791, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 1262.239990234375, "completions/mean_terminated_length": 1246.2041015625, "completions/min_length": 687.0, "completions/min_terminated_length": 687.0, "entropy": 0.21845731139183044, "epoch": 0.07065217391304347, "frac_reward_zero_std": 0.0, "grad_norm": 0.019048262387514114, "learning_rate": 5e-05, "loss": 0.11241159588098526, "num_tokens": 2247181.0, "reward": -0.056328125298023224, "reward_std": 0.48785343766212463, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.6163281202316284, "rewards/length_penalty/std": 0.16271910071372986, "sampling/importance_sampling_ratio/max": 1.8472636938095093, "sampling/importance_sampling_ratio/mean": 0.9921180605888367, "sampling/importance_sampling_ratio/min": 0.45372647047042847, "sampling/sampling_logp_difference/max": 0.7902607917785645, "sampling/sampling_logp_difference/mean": 0.015436006709933281, "step": 26, "step_time": 26.193258133949712 }, { "clip_ratio/high_max": 0.00045611696768901313, "clip_ratio/high_mean": 0.00045611696768901313, "clip_ratio/low_mean": 0.00010173475166084245, "clip_ratio/low_min": 0.00010173475166084245, "clip_ratio/region_mean": 0.0005578517040703446, "completions/clipped_ratio": 0.3999999761581421, "completions/max_length": 2048.0, "completions/max_terminated_length": 2031.0, "completions/mean_length": 1539.419921875, "completions/mean_terminated_length": 1200.36669921875, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.2073049932718277, "epoch": 0.07336956521739131, "frac_reward_zero_std": 0.0, "grad_norm": 0.0178088266402483, "learning_rate": 5e-05, "loss": 0.05440214276313782, "num_tokens": 2326402.0, "reward": -0.1516699194908142, "reward_std": 0.7180583477020264, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.7516699433326721, "rewards/length_penalty/std": 0.26075154542922974, "sampling/importance_sampling_ratio/max": 2.0076329708099365, "sampling/importance_sampling_ratio/mean": 0.9929074645042419, "sampling/importance_sampling_ratio/min": 0.3890666961669922, "sampling/sampling_logp_difference/max": 0.9440045356750488, "sampling/sampling_logp_difference/mean": 0.014081662520766258, "step": 27, "step_time": 26.73872535675764 }, { "clip_ratio/high_max": 0.0007280473771970719, "clip_ratio/high_mean": 0.0007280473771970719, "clip_ratio/low_mean": 6.700347003061325e-05, "clip_ratio/low_min": 6.700347003061325e-05, "clip_ratio/region_mean": 0.0007950508443173021, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1852.0, "completions/mean_length": 1207.4599609375, "completions/mean_terminated_length": 1190.30615234375, "completions/min_length": 638.0, "completions/min_terminated_length": 638.0, "entropy": 0.1798260122537613, "epoch": 0.07608695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.016198398545384407, "learning_rate": 5e-05, "loss": 0.09480622410774231, "num_tokens": 2390135.0, "reward": 0.2104199230670929, "reward_std": 0.4322068691253662, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.5895800590515137, "rewards/length_penalty/std": 0.1593828648328781, "sampling/importance_sampling_ratio/max": 1.6265084743499756, "sampling/importance_sampling_ratio/mean": 0.993723452091217, "sampling/importance_sampling_ratio/min": 0.38461166620254517, "sampling/sampling_logp_difference/max": 0.9555211067199707, "sampling/sampling_logp_difference/mean": 0.012959480285644531, "step": 28, "step_time": 25.803300652187318 }, { "clip_ratio/high_max": 0.0007129237637855113, "clip_ratio/high_mean": 0.0007129237637855113, "clip_ratio/low_mean": 0.00013153162435628474, "clip_ratio/low_min": 0.00013153162435628474, "clip_ratio/region_mean": 0.000844455393962562, "completions/clipped_ratio": 0.3799999952316284, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 1605.52001953125, "completions/mean_terminated_length": 1334.322509765625, "completions/min_length": 545.0, "completions/min_terminated_length": 545.0, "entropy": 0.23342860341072083, "epoch": 0.07880434782608696, "frac_reward_zero_std": 0.0, "grad_norm": 0.019112825393676758, "learning_rate": 5e-05, "loss": 0.08988890796899796, "num_tokens": 2473851.0, "reward": -0.08394531160593033, "reward_std": 0.643122673034668, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.7839453220367432, "rewards/length_penalty/std": 0.2589260935783386, "sampling/importance_sampling_ratio/max": 2.976001739501953, "sampling/importance_sampling_ratio/mean": 0.9917635917663574, "sampling/importance_sampling_ratio/min": 0.3071954846382141, "sampling/sampling_logp_difference/max": 1.180271029472351, "sampling/sampling_logp_difference/mean": 0.016178693622350693, "step": 29, "step_time": 27.464959647972137 }, { "clip_ratio/high_max": 0.0007170933298766613, "clip_ratio/high_mean": 0.0007170933298766613, "clip_ratio/low_mean": 7.078182243276387e-05, "clip_ratio/low_min": 7.078182243276387e-05, "clip_ratio/region_mean": 0.0007878751552198082, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1948.0, "completions/mean_length": 1215.739990234375, "completions/mean_terminated_length": 1102.25, "completions/min_length": 676.0, "completions/min_terminated_length": 676.0, "entropy": 0.1600349336862564, "epoch": 0.08152173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.01690404675900936, "learning_rate": 5e-05, "loss": 0.10917580127716064, "num_tokens": 2536428.0, "reward": 0.286376953125, "reward_std": 0.49757134914398193, "rewards/correctness/mean": 0.8799999952316284, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.5936230421066284, "rewards/length_penalty/std": 0.20075169205665588, "sampling/importance_sampling_ratio/max": 2.474348306655884, "sampling/importance_sampling_ratio/mean": 0.9940484762191772, "sampling/importance_sampling_ratio/min": 0.2940029501914978, "sampling/sampling_logp_difference/max": 1.224165439605713, "sampling/sampling_logp_difference/mean": 0.012239653617143631, "step": 30, "step_time": 25.22584995208308 }, { "clip_ratio/high_max": 0.000856519874650985, "clip_ratio/high_mean": 0.000856519874650985, "clip_ratio/low_mean": 8.000027883099392e-05, "clip_ratio/low_min": 8.000027883099392e-05, "clip_ratio/region_mean": 0.0009365201694890857, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1861.0, "completions/mean_length": 1463.8800048828125, "completions/mean_terminated_length": 1317.8499755859375, "completions/min_length": 510.0, "completions/min_terminated_length": 510.0, "entropy": 0.17065878212451935, "epoch": 0.08423913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.01730084978044033, "learning_rate": 5e-05, "loss": 0.12122836709022522, "num_tokens": 2612752.0, "reward": -0.29478514194488525, "reward_std": 0.5411331653594971, "rewards/correctness/mean": 0.41999998688697815, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.7147851586341858, "rewards/length_penalty/std": 0.20178328454494476, "sampling/importance_sampling_ratio/max": 2.579355478286743, "sampling/importance_sampling_ratio/mean": 0.9938713908195496, "sampling/importance_sampling_ratio/min": 0.2596469223499298, "sampling/sampling_logp_difference/max": 1.3484325408935547, "sampling/sampling_logp_difference/mean": 0.012738464400172234, "step": 31, "step_time": 27.176139670424163 }, { "clip_ratio/high_max": 0.0008451115863863379, "clip_ratio/high_mean": 0.0008451115863863379, "clip_ratio/low_mean": 0.00011503964924486354, "clip_ratio/low_min": 0.00011503964924486354, "clip_ratio/region_mean": 0.0009601512225344777, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 2000.0, "completions/mean_length": 1191.3399658203125, "completions/mean_terminated_length": 1028.1666259765625, "completions/min_length": 552.0, "completions/min_terminated_length": 552.0, "entropy": 0.21862303614616393, "epoch": 0.08695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.01855313777923584, "learning_rate": 5e-05, "loss": 0.1444239616394043, "num_tokens": 2674249.0, "reward": 0.23829101026058197, "reward_std": 0.5837259888648987, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.5817089676856995, "rewards/length_penalty/std": 0.2382737696170807, "sampling/importance_sampling_ratio/max": 1.9388597011566162, "sampling/importance_sampling_ratio/mean": 0.9922151565551758, "sampling/importance_sampling_ratio/min": 0.24645483493804932, "sampling/sampling_logp_difference/max": 1.4005765914916992, "sampling/sampling_logp_difference/mean": 0.0162365585565567, "step": 32, "step_time": 25.234165051020682 }, { "clip_ratio/high_max": 0.0009740249835886062, "clip_ratio/high_mean": 0.0009740249835886062, "clip_ratio/low_mean": 5.943443829892203e-05, "clip_ratio/low_min": 5.943443829892203e-05, "clip_ratio/region_mean": 0.001033459440805018, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1974.0, "completions/mean_length": 1382.0599365234375, "completions/mean_terminated_length": 1148.0810546875, "completions/min_length": 706.0, "completions/min_terminated_length": 706.0, "entropy": 0.20403595566749572, "epoch": 0.08967391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.016193265095353127, "learning_rate": 5e-05, "loss": 0.05690198391675949, "num_tokens": 2746992.0, "reward": -0.09483398497104645, "reward_std": 0.6409366726875305, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.6748340129852295, "rewards/length_penalty/std": 0.2429748922586441, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9926742315292358, "sampling/importance_sampling_ratio/min": 0.1798250824213028, "sampling/sampling_logp_difference/max": 1.7157707214355469, "sampling/sampling_logp_difference/mean": 0.014851015992462635, "step": 33, "step_time": 26.865770722040907 }, { "clip_ratio/high_max": 0.0012180538265965878, "clip_ratio/high_mean": 0.0012180538265965878, "clip_ratio/low_mean": 2.9444240499287844e-05, "clip_ratio/low_min": 2.9444240499287844e-05, "clip_ratio/region_mean": 0.0012474980554543435, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 1734.0, "completions/mean_length": 1317.179931640625, "completions/mean_terminated_length": 1111.05126953125, "completions/min_length": 647.0, "completions/min_terminated_length": 647.0, "entropy": 0.22754225134849548, "epoch": 0.09239130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.020097807049751282, "learning_rate": 5e-05, "loss": 0.0710531622171402, "num_tokens": 2816741.0, "reward": 0.11684569716453552, "reward_std": 0.61951744556427, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.6431543231010437, "rewards/length_penalty/std": 0.2254319041967392, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9915940761566162, "sampling/importance_sampling_ratio/min": 0.23836448788642883, "sampling/sampling_logp_difference/max": 1.4339543581008911, "sampling/sampling_logp_difference/mean": 0.01691325753927231, "step": 34, "step_time": 26.204810510622337 }, { "clip_ratio/high_max": 0.0008569001860450953, "clip_ratio/high_mean": 0.0008569001860450953, "clip_ratio/low_mean": 0.00013024549843976274, "clip_ratio/low_min": 0.00013024549843976274, "clip_ratio/region_mean": 0.0009871456888504327, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 2030.0, "completions/mean_length": 1345.1400146484375, "completions/mean_terminated_length": 1211.261962890625, "completions/min_length": 632.0, "completions/min_terminated_length": 632.0, "entropy": 0.26689461171627044, "epoch": 0.09510869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.0222671777009964, "learning_rate": 5e-05, "loss": 0.15181683003902435, "num_tokens": 2886728.0, "reward": 0.1431933492422104, "reward_std": 0.5873222351074219, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.6568066477775574, "rewards/length_penalty/std": 0.22363705933094025, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.990323543548584, "sampling/importance_sampling_ratio/min": 0.16977547109127045, "sampling/sampling_logp_difference/max": 1.7732784748077393, "sampling/sampling_logp_difference/mean": 0.01977042853832245, "step": 35, "step_time": 25.99194298312068 }, { "clip_ratio/high_max": 0.0007262625789735466, "clip_ratio/high_mean": 0.0007262625789735466, "clip_ratio/low_mean": 0.0001643200113903731, "clip_ratio/low_min": 0.0001643200113903731, "clip_ratio/region_mean": 0.0008905825903639198, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1975.0, "completions/mean_length": 1117.199951171875, "completions/mean_terminated_length": 990.2727661132812, "completions/min_length": 532.0, "completions/min_terminated_length": 532.0, "entropy": 0.1896991550922394, "epoch": 0.09782608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.01747210882604122, "learning_rate": 5e-05, "loss": 0.058442991226911545, "num_tokens": 2945058.0, "reward": 0.33449217677116394, "reward_std": 0.5202338695526123, "rewards/correctness/mean": 0.8799999952316284, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.5455077886581421, "rewards/length_penalty/std": 0.22712095081806183, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930076599121094, "sampling/importance_sampling_ratio/min": 0.1576518714427948, "sampling/sampling_logp_difference/max": 1.8473659753799438, "sampling/sampling_logp_difference/mean": 0.01649167202413082, "step": 36, "step_time": 25.64875900489278 }, { "clip_ratio/high_max": 0.0007515452452935278, "clip_ratio/high_mean": 0.0007515452452935278, "clip_ratio/low_mean": 0.00017000650841509924, "clip_ratio/low_min": 0.00017000650841509924, "clip_ratio/region_mean": 0.0009215517551638186, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 1888.0, "completions/mean_length": 1225.93994140625, "completions/mean_terminated_length": 994.0769653320312, "completions/min_length": 605.0, "completions/min_terminated_length": 605.0, "entropy": 0.2357550859451294, "epoch": 0.10054347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.019464492797851562, "learning_rate": 5e-05, "loss": 0.09758821129798889, "num_tokens": 3009875.0, "reward": -0.01860351487994194, "reward_std": 0.6543649435043335, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.5986034870147705, "rewards/length_penalty/std": 0.24733325839042664, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9915767908096313, "sampling/importance_sampling_ratio/min": 0.23241811990737915, "sampling/sampling_logp_difference/max": 1.4991412162780762, "sampling/sampling_logp_difference/mean": 0.01839047111570835, "step": 37, "step_time": 26.028307612985373 }, { "clip_ratio/high_max": 0.0007471224380424246, "clip_ratio/high_mean": 0.0007471224380424246, "clip_ratio/low_mean": 5.083655851194635e-05, "clip_ratio/low_min": 5.083655851194635e-05, "clip_ratio/region_mean": 0.0007979590183822439, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1990.0, "completions/mean_length": 1287.699951171875, "completions/mean_terminated_length": 1203.2222900390625, "completions/min_length": 667.0, "completions/min_terminated_length": 667.0, "entropy": 0.1729876458644867, "epoch": 0.10326086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.01705159805715084, "learning_rate": 5e-05, "loss": 0.07221940904855728, "num_tokens": 3077650.0, "reward": 0.25124022364616394, "reward_std": 0.4530826508998871, "rewards/correctness/mean": 0.8799999952316284, "rewards/correctness/std": 0.32826074957847595, "rewards/length_penalty/mean": -0.6287597417831421, "rewards/length_penalty/std": 0.2424236387014389, "sampling/importance_sampling_ratio/max": 2.784482002258301, "sampling/importance_sampling_ratio/mean": 0.9937966465950012, "sampling/importance_sampling_ratio/min": 0.0835733637213707, "sampling/sampling_logp_difference/max": 2.4820303916931152, "sampling/sampling_logp_difference/mean": 0.014295706525444984, "step": 38, "step_time": 26.3849402081687 }, { "clip_ratio/high_max": 0.00047908292326610536, "clip_ratio/high_mean": 0.00047908292326610536, "clip_ratio/low_mean": 6.898379942867904e-05, "clip_ratio/low_min": 6.898379942867904e-05, "clip_ratio/region_mean": 0.0005480667285155505, "completions/clipped_ratio": 0.17999999225139618, "completions/max_length": 2048.0, "completions/max_terminated_length": 1733.0, "completions/mean_length": 1131.1400146484375, "completions/mean_terminated_length": 929.8779907226562, "completions/min_length": 473.0, "completions/min_terminated_length": 473.0, "entropy": 0.18207353055477143, "epoch": 0.10597826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.02022084966301918, "learning_rate": 5e-05, "loss": 0.05686383694410324, "num_tokens": 3136697.0, "reward": 0.06768554449081421, "reward_std": 0.7044277191162109, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.5523144602775574, "rewards/length_penalty/std": 0.25435444712638855, "sampling/importance_sampling_ratio/max": 2.187497615814209, "sampling/importance_sampling_ratio/mean": 0.993350088596344, "sampling/importance_sampling_ratio/min": 0.27275171875953674, "sampling/sampling_logp_difference/max": 1.2991933822631836, "sampling/sampling_logp_difference/mean": 0.014788741245865822, "step": 39, "step_time": 24.97533990885131 }, { "clip_ratio/high_max": 0.0007706037955358625, "clip_ratio/high_mean": 0.0007706037955358625, "clip_ratio/low_mean": 9.303098486270755e-05, "clip_ratio/low_min": 9.303098486270755e-05, "clip_ratio/region_mean": 0.0008636347949504853, "completions/clipped_ratio": 0.0, "completions/max_length": 1812.0, "completions/max_terminated_length": 1812.0, "completions/mean_length": 870.8999633789062, "completions/mean_terminated_length": 870.8999633789062, "completions/min_length": 654.0, "completions/min_terminated_length": 654.0, "entropy": 0.1895820140838623, "epoch": 0.10869565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.014811172150075436, "learning_rate": 5e-05, "loss": 0.06211411952972412, "num_tokens": 3182962.0, "reward": 0.574755847454071, "reward_std": 0.1076437383890152, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.42524415254592896, "rewards/length_penalty/std": 0.1076437383890152, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.993091881275177, "sampling/importance_sampling_ratio/min": 0.11348514258861542, "sampling/sampling_logp_difference/max": 2.1760833263397217, "sampling/sampling_logp_difference/mean": 0.017822640016674995, "step": 40, "step_time": 21.239220429910347 }, { "clip_ratio/high_max": 0.000729369098553434, "clip_ratio/high_mean": 0.000729369098553434, "clip_ratio/low_mean": 6.442431767936796e-05, "clip_ratio/low_min": 6.442431767936796e-05, "clip_ratio/region_mean": 0.0007937934075016529, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1935.0, "completions/mean_length": 1214.1199951171875, "completions/mean_terminated_length": 1100.4091796875, "completions/min_length": 598.0, "completions/min_terminated_length": 598.0, "entropy": 0.20386375486850739, "epoch": 0.11141304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.0198789294809103, "learning_rate": 5e-05, "loss": 0.09383414685726166, "num_tokens": 3247408.0, "reward": -0.05283202975988388, "reward_std": 0.612436830997467, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.5928320288658142, "rewards/length_penalty/std": 0.20813514292240143, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9923471212387085, "sampling/importance_sampling_ratio/min": 0.11420464515686035, "sampling/sampling_logp_difference/max": 2.1697633266448975, "sampling/sampling_logp_difference/mean": 0.017043285071849823, "step": 41, "step_time": 25.912487969035283 }, { "clip_ratio/high_max": 0.0005029736028518527, "clip_ratio/high_mean": 0.0005029736028518527, "clip_ratio/low_mean": 0.00012810062034986913, "clip_ratio/low_min": 0.00012810062034986913, "clip_ratio/region_mean": 0.0006310742348432541, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1992.0, "completions/mean_length": 1178.3599853515625, "completions/mean_terminated_length": 960.9500122070312, "completions/min_length": 471.0, "completions/min_terminated_length": 471.0, "entropy": 0.20278692245483398, "epoch": 0.11413043478260869, "frac_reward_zero_std": 0.0, "grad_norm": 0.01870260015130043, "learning_rate": 5e-05, "loss": 0.03833295777440071, "num_tokens": 3309316.0, "reward": 0.22462889552116394, "reward_std": 0.6349654197692871, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5753710865974426, "rewards/length_penalty/std": 0.2908514440059662, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9925538301467896, "sampling/importance_sampling_ratio/min": 0.16318759322166443, "sampling/sampling_logp_difference/max": 1.8128548860549927, "sampling/sampling_logp_difference/mean": 0.01731746457517147, "step": 42, "step_time": 26.0047237186227 }, { "clip_ratio/high_max": 0.001067713089287281, "clip_ratio/high_mean": 0.001067713089287281, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001067713089287281, "completions/clipped_ratio": 0.0, "completions/max_length": 1500.0, "completions/max_terminated_length": 1500.0, "completions/mean_length": 777.4400024414062, "completions/mean_terminated_length": 777.4400024414062, "completions/min_length": 481.0, "completions/min_terminated_length": 481.0, "entropy": 0.2059490293264389, "epoch": 0.11684782608695653, "frac_reward_zero_std": 0.0, "grad_norm": 0.01624198630452156, "learning_rate": 5e-05, "loss": 0.05506262555718422, "num_tokens": 3350158.0, "reward": 0.600390613079071, "reward_std": 0.18859446048736572, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.3796093761920929, "rewards/length_penalty/std": 0.12124241143465042, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9921716451644897, "sampling/importance_sampling_ratio/min": 0.048016149550676346, "sampling/sampling_logp_difference/max": 3.0362179279327393, "sampling/sampling_logp_difference/mean": 0.021691124886274338, "step": 43, "step_time": 17.796076989499852 }, { "clip_ratio/high_max": 0.0006964510248508304, "clip_ratio/high_mean": 0.0006964510248508304, "clip_ratio/low_mean": 0.00017296440637437627, "clip_ratio/low_min": 0.00017296440637437627, "clip_ratio/region_mean": 0.0008694154501426965, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1803.0, "completions/mean_length": 1044.780029296875, "completions/mean_terminated_length": 1024.30615234375, "completions/min_length": 441.0, "completions/min_terminated_length": 441.0, "entropy": 0.1765652060508728, "epoch": 0.11956521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.015211697667837143, "learning_rate": 5e-05, "loss": 0.09405821561813354, "num_tokens": 3405397.0, "reward": 0.4698534905910492, "reward_std": 0.28211453557014465, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.5101464986801147, "rewards/length_penalty/std": 0.19898390769958496, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9933960437774658, "sampling/importance_sampling_ratio/min": 0.05977986007928848, "sampling/sampling_logp_difference/max": 2.8170864582061768, "sampling/sampling_logp_difference/mean": 0.016818420961499214, "step": 44, "step_time": 24.691332526504993 }, { "clip_ratio/high_max": 0.0007825112727005035, "clip_ratio/high_mean": 0.0007825112727005035, "clip_ratio/low_mean": 3.903874894604087e-05, "clip_ratio/low_min": 3.903874894604087e-05, "clip_ratio/region_mean": 0.0008215500216465444, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1795.0, "completions/mean_length": 1010.0199584960938, "completions/mean_terminated_length": 966.7708740234375, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "entropy": 0.2038260132074356, "epoch": 0.12228260869565218, "frac_reward_zero_std": 0.0, "grad_norm": 0.016963917762041092, "learning_rate": 5e-05, "loss": 0.05373412370681763, "num_tokens": 3458838.0, "reward": 0.28682616353034973, "reward_std": 0.5763336420059204, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.49317383766174316, "rewards/length_penalty/std": 0.2312302142381668, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9924213886260986, "sampling/importance_sampling_ratio/min": 0.06157507002353668, "sampling/sampling_logp_difference/max": 2.7874982357025146, "sampling/sampling_logp_difference/mean": 0.018136093392968178, "step": 45, "step_time": 25.164567972067744 }, { "clip_ratio/high_max": 0.00039000823890091854, "clip_ratio/high_mean": 0.00039000823890091854, "clip_ratio/low_mean": 7.269879861269147e-05, "clip_ratio/low_min": 7.269879861269147e-05, "clip_ratio/region_mean": 0.0004627070476999506, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1188.0, "completions/mean_length": 1086.6400146484375, "completions/mean_terminated_length": 846.2999877929688, "completions/min_length": 463.0, "completions/min_terminated_length": 463.0, "entropy": 0.22776823937892915, "epoch": 0.125, "frac_reward_zero_std": 0.0, "grad_norm": 0.015369636937975883, "learning_rate": 5e-05, "loss": 0.04503121227025986, "num_tokens": 3516680.0, "reward": 0.2694140672683716, "reward_std": 0.6474911570549011, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5305859446525574, "rewards/length_penalty/std": 0.25373587012290955, "sampling/importance_sampling_ratio/max": 2.9789910316467285, "sampling/importance_sampling_ratio/mean": 0.9916025996208191, "sampling/importance_sampling_ratio/min": 0.04188171774148941, "sampling/sampling_logp_difference/max": 3.172905921936035, "sampling/sampling_logp_difference/mean": 0.019508102908730507, "step": 46, "step_time": 25.130053090164438 }, { "clip_ratio/high_max": 0.0006507517013233155, "clip_ratio/high_mean": 0.0006507517013233155, "clip_ratio/low_mean": 0.00011765561066567898, "clip_ratio/low_min": 0.00011765561066567898, "clip_ratio/region_mean": 0.0007684073178097605, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1485.0, "completions/mean_length": 826.6199951171875, "completions/mean_terminated_length": 775.7291870117188, "completions/min_length": 500.0, "completions/min_terminated_length": 500.0, "entropy": 0.175468310713768, "epoch": 0.12771739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.01958637870848179, "learning_rate": 5e-05, "loss": 0.06920386850833893, "num_tokens": 3560401.0, "reward": 0.5163769125938416, "reward_std": 0.41254669427871704, "rewards/correctness/mean": 0.9200000166893005, "rewards/correctness/std": 0.27404752373695374, "rewards/length_penalty/mean": -0.4036230444908142, "rewards/length_penalty/std": 0.16869166493415833, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.993443489074707, "sampling/importance_sampling_ratio/min": 0.13633184134960175, "sampling/sampling_logp_difference/max": 1.9926633834838867, "sampling/sampling_logp_difference/mean": 0.01789533719420433, "step": 47, "step_time": 23.82672528317198 }, { "clip_ratio/high_max": 0.0005032541987020523, "clip_ratio/high_mean": 0.0005032541987020523, "clip_ratio/low_mean": 6.262429669732228e-05, "clip_ratio/low_min": 6.262429669732228e-05, "clip_ratio/region_mean": 0.0005658784910337999, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 1960.0, "completions/mean_length": 1017.1199951171875, "completions/mean_terminated_length": 927.478271484375, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "entropy": 0.1553356945514679, "epoch": 0.13043478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.017065836116671562, "learning_rate": 5e-05, "loss": 0.05763271450996399, "num_tokens": 3614217.0, "reward": 0.18335936963558197, "reward_std": 0.617246687412262, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.4966406226158142, "rewards/length_penalty/std": 0.2483815848827362, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9945675134658813, "sampling/importance_sampling_ratio/min": 0.1440926343202591, "sampling/sampling_logp_difference/max": 2.313066005706787, "sampling/sampling_logp_difference/mean": 0.014977255836129189, "step": 48, "step_time": 25.213895262451842 }, { "clip_ratio/high_max": 0.0016091061057522892, "clip_ratio/high_mean": 0.0016091061057522892, "clip_ratio/low_mean": 4.549590521492064e-05, "clip_ratio/low_min": 4.549590521492064e-05, "clip_ratio/region_mean": 0.0016546020051464438, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1164.0, "completions/mean_length": 972.0799560546875, "completions/mean_terminated_length": 703.1000366210938, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.24074310958385467, "epoch": 0.1331521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.01453313510864973, "learning_rate": 5e-05, "loss": 0.04727436602115631, "num_tokens": 3665511.0, "reward": 0.3253515660762787, "reward_std": 0.6757630109786987, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.47464844584465027, "rewards/length_penalty/std": 0.2810002267360687, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.991299033164978, "sampling/importance_sampling_ratio/min": 0.12447156012058258, "sampling/sampling_logp_difference/max": 2.0836780071258545, "sampling/sampling_logp_difference/mean": 0.021517034620046616, "step": 49, "step_time": 24.520182807929814 }, { "clip_ratio/high_max": 0.0003247870015911758, "clip_ratio/high_mean": 0.0003247870015911758, "clip_ratio/low_mean": 0.00028453292179619893, "clip_ratio/low_min": 0.00028453292179619893, "clip_ratio/region_mean": 0.0006093199306633323, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 912.2799682617188, "completions/mean_terminated_length": 628.3500366210938, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.25229776501655576, "epoch": 0.1358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.013235251419246197, "learning_rate": 5e-05, "loss": 0.02668197639286518, "num_tokens": 3713535.0, "reward": 0.15455077588558197, "reward_std": 0.7314342856407166, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.44544923305511475, "rewards/length_penalty/std": 0.29221686720848083, "sampling/importance_sampling_ratio/max": 2.721278667449951, "sampling/importance_sampling_ratio/mean": 0.9899016618728638, "sampling/importance_sampling_ratio/min": 0.042893968522548676, "sampling/sampling_logp_difference/max": 3.14902400970459, "sampling/sampling_logp_difference/mean": 0.022883890196681023, "step": 50, "step_time": 24.369463649811223 }, { "clip_ratio/high_max": 0.000643822574056685, "clip_ratio/high_mean": 0.000643822574056685, "clip_ratio/low_mean": 6.33914431091398e-05, "clip_ratio/low_min": 6.33914431091398e-05, "clip_ratio/region_mean": 0.0007072140229865909, "completions/clipped_ratio": 0.0, "completions/max_length": 1970.0, "completions/max_terminated_length": 1970.0, "completions/mean_length": 766.9599609375, "completions/mean_terminated_length": 766.9599609375, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "entropy": 0.16464298963546753, "epoch": 0.13858695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.015319639816880226, "learning_rate": 5e-05, "loss": 0.06438955664634705, "num_tokens": 3754393.0, "reward": 0.6255077719688416, "reward_std": 0.1482277661561966, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.37449219822883606, "rewards/length_penalty/std": 0.1482277661561966, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9936192035675049, "sampling/importance_sampling_ratio/min": 0.11222824454307556, "sampling/sampling_logp_difference/max": 2.187220573425293, "sampling/sampling_logp_difference/mean": 0.018299877643585205, "step": 51, "step_time": 22.948548633838072 }, { "clip_ratio/high_max": 0.0007740166562143713, "clip_ratio/high_mean": 0.0007740166562143713, "clip_ratio/low_mean": 0.0001623515330720693, "clip_ratio/low_min": 0.0001623515330720693, "clip_ratio/region_mean": 0.0009363682067487389, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 700.0799560546875, "completions/mean_terminated_length": 700.0799560546875, "completions/min_length": 454.0, "completions/min_terminated_length": 454.0, "entropy": 0.15744704604148865, "epoch": 0.14130434782608695, "frac_reward_zero_std": 0.0, "grad_norm": 0.012992658652365208, "learning_rate": 5e-05, "loss": 0.04073449596762657, "num_tokens": 3791657.0, "reward": 0.4581640660762787, "reward_std": 0.41504931449890137, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.34183594584465027, "rewards/length_penalty/std": 0.11200068145990372, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9942061305046082, "sampling/importance_sampling_ratio/min": 0.15024611353874207, "sampling/sampling_logp_difference/max": 1.8954805135726929, "sampling/sampling_logp_difference/mean": 0.019554542377591133, "step": 52, "step_time": 16.728716250276193 }, { "clip_ratio/high_max": 0.0006250745645957068, "clip_ratio/high_mean": 0.0006250745645957068, "clip_ratio/low_mean": 2.658160519786179e-05, "clip_ratio/low_min": 2.658160519786179e-05, "clip_ratio/region_mean": 0.0006516561697935686, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1354.0, "completions/mean_length": 807.3599853515625, "completions/mean_terminated_length": 755.6666870117188, "completions/min_length": 397.0, "completions/min_terminated_length": 397.0, "entropy": 0.1829791933298111, "epoch": 0.14402173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.021517958492040634, "learning_rate": 5e-05, "loss": 0.10340322554111481, "num_tokens": 3834745.0, "reward": 0.3657812476158142, "reward_std": 0.571151077747345, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.3942187428474426, "rewards/length_penalty/std": 0.1765245497226715, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9934973120689392, "sampling/importance_sampling_ratio/min": 0.05403505638241768, "sampling/sampling_logp_difference/max": 2.9181222915649414, "sampling/sampling_logp_difference/mean": 0.021358292549848557, "step": 53, "step_time": 23.810034946305677 }, { "clip_ratio/high_max": 0.0007082508644089102, "clip_ratio/high_mean": 0.0007082508644089102, "clip_ratio/low_mean": 0.00010528063430683687, "clip_ratio/low_min": 0.00010528063430683687, "clip_ratio/region_mean": 0.0008135314856190234, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1746.0, "completions/mean_length": 1111.3199462890625, "completions/mean_terminated_length": 983.5909423828125, "completions/min_length": 512.0, "completions/min_terminated_length": 512.0, "entropy": 0.1832393229007721, "epoch": 0.14673913043478262, "frac_reward_zero_std": 0.0, "grad_norm": 0.019397616386413574, "learning_rate": 5e-05, "loss": 0.11991956830024719, "num_tokens": 3894131.0, "reward": 0.31736326217651367, "reward_std": 0.5375475883483887, "rewards/correctness/mean": 0.8600000143051147, "rewards/correctness/std": 0.3505098223686218, "rewards/length_penalty/mean": -0.5426366925239563, "rewards/length_penalty/std": 0.23527614772319794, "sampling/importance_sampling_ratio/max": 2.2643704414367676, "sampling/importance_sampling_ratio/mean": 0.9930412769317627, "sampling/importance_sampling_ratio/min": 0.04445350170135498, "sampling/sampling_logp_difference/max": 3.113311529159546, "sampling/sampling_logp_difference/mean": 0.01802951470017433, "step": 54, "step_time": 25.670479513239115 }, { "clip_ratio/high_max": 0.000749661133158952, "clip_ratio/high_mean": 0.000749661133158952, "clip_ratio/low_mean": 0.00012117865699110553, "clip_ratio/low_min": 0.00012117865699110553, "clip_ratio/region_mean": 0.0008708397857844829, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 2039.0, "completions/mean_length": 1065.1199951171875, "completions/mean_terminated_length": 931.0909423828125, "completions/min_length": 480.0, "completions/min_terminated_length": 480.0, "entropy": 0.22501038908958435, "epoch": 0.14945652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.021781066432595253, "learning_rate": 5e-05, "loss": 0.07857176661491394, "num_tokens": 3950277.0, "reward": 0.3199218809604645, "reward_std": 0.5881868600845337, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.5200781226158142, "rewards/length_penalty/std": 0.28003421425819397, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9916319251060486, "sampling/importance_sampling_ratio/min": 0.04320801421999931, "sampling/sampling_logp_difference/max": 3.1417293548583984, "sampling/sampling_logp_difference/mean": 0.021369094029068947, "step": 55, "step_time": 24.98296606983058 }, { "clip_ratio/high_max": 0.0002201356299337931, "clip_ratio/high_mean": 0.0002201356299337931, "clip_ratio/low_mean": 0.0001502036669990048, "clip_ratio/low_min": 0.0001502036669990048, "clip_ratio/region_mean": 0.000370339305663947, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1660.0, "completions/mean_length": 928.5, "completions/mean_terminated_length": 648.625, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.2072504162788391, "epoch": 0.15217391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.013964945450425148, "learning_rate": 5e-05, "loss": 0.05491746589541435, "num_tokens": 3999772.0, "reward": 0.14663085341453552, "reward_std": 0.6885973215103149, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.453369140625, "rewards/length_penalty/std": 0.30666640400886536, "sampling/importance_sampling_ratio/max": 2.906921625137329, "sampling/importance_sampling_ratio/mean": 0.9920700788497925, "sampling/importance_sampling_ratio/min": 0.03219592198729515, "sampling/sampling_logp_difference/max": 3.435915470123291, "sampling/sampling_logp_difference/mean": 0.02083013206720352, "step": 56, "step_time": 24.479757665889338 }, { "clip_ratio/high_max": 0.001786479854490608, "clip_ratio/high_mean": 0.001786479854490608, "clip_ratio/low_mean": 2.5886617368087173e-05, "clip_ratio/low_min": 2.5886617368087173e-05, "clip_ratio/region_mean": 0.0018123664776794612, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1872.0, "completions/mean_length": 1050.699951171875, "completions/mean_terminated_length": 801.375, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.2278908148407936, "epoch": 0.15489130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.014048721641302109, "learning_rate": 5e-05, "loss": 0.037405215203762054, "num_tokens": 4055547.0, "reward": 0.2869628965854645, "reward_std": 0.6820490956306458, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5130370855331421, "rewards/length_penalty/std": 0.32119351625442505, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.991212785243988, "sampling/importance_sampling_ratio/min": 0.018531763926148415, "sampling/sampling_logp_difference/max": 3.988269090652466, "sampling/sampling_logp_difference/mean": 0.021337997168302536, "step": 57, "step_time": 24.977965974714607 }, { "clip_ratio/high_max": 0.0005700029694708064, "clip_ratio/high_mean": 0.0005700029694708064, "clip_ratio/low_mean": 0.00015229834825731815, "clip_ratio/low_min": 0.00015229834825731815, "clip_ratio/region_mean": 0.0007223013206385076, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1868.0, "completions/mean_length": 1198.6199951171875, "completions/mean_terminated_length": 900.189208984375, "completions/min_length": 466.0, "completions/min_terminated_length": 466.0, "entropy": 0.255254665017128, "epoch": 0.15760869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.01905212551355362, "learning_rate": 5e-05, "loss": 0.09029528498649597, "num_tokens": 4119288.0, "reward": -0.0052636717446148396, "reward_std": 0.7467082142829895, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.5852636694908142, "rewards/length_penalty/std": 0.3044397234916687, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9905498623847961, "sampling/importance_sampling_ratio/min": 0.05557810142636299, "sampling/sampling_logp_difference/max": 2.8899660110473633, "sampling/sampling_logp_difference/mean": 0.023080680519342422, "step": 58, "step_time": 26.597216782160103 }, { "clip_ratio/high_max": 0.0008118896570522338, "clip_ratio/high_mean": 0.0008118896570522338, "clip_ratio/low_mean": 0.00015391591878142208, "clip_ratio/low_min": 0.00015391591878142208, "clip_ratio/region_mean": 0.0009658055671025068, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 677.4199829101562, "completions/mean_terminated_length": 649.448974609375, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.14910456240177156, "epoch": 0.16032608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.016270486637949944, "learning_rate": 5e-05, "loss": 0.04443974792957306, "num_tokens": 4155869.0, "reward": 0.5692285299301147, "reward_std": 0.390527606010437, "rewards/correctness/mean": 0.8999999761581421, "rewards/correctness/std": 0.30304574966430664, "rewards/length_penalty/mean": -0.33077147603034973, "rewards/length_penalty/std": 0.146135151386261, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9942877888679504, "sampling/importance_sampling_ratio/min": 0.08243148028850555, "sampling/sampling_logp_difference/max": 2.4957878589630127, "sampling/sampling_logp_difference/mean": 0.019903726875782013, "step": 59, "step_time": 22.8233163098339 }, { "clip_ratio/high_max": 0.0006108953821239993, "clip_ratio/high_mean": 0.0006108953821239993, "clip_ratio/low_mean": 8.367137343157083e-05, "clip_ratio/low_min": 8.367137343157083e-05, "clip_ratio/region_mean": 0.0006945667759282514, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 740.719970703125, "completions/mean_terminated_length": 740.719970703125, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.1358457773923874, "epoch": 0.16304347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.01377896312624216, "learning_rate": 5e-05, "loss": 0.04055830091238022, "num_tokens": 4195715.0, "reward": 0.018320312723517418, "reward_std": 0.4930758476257324, "rewards/correctness/mean": 0.3799999952316284, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.36167967319488525, "rewards/length_penalty/std": 0.10894857347011566, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9946678876876831, "sampling/importance_sampling_ratio/min": 0.06510572135448456, "sampling/sampling_logp_difference/max": 2.7317428588867188, "sampling/sampling_logp_difference/mean": 0.01834091730415821, "step": 60, "step_time": 14.378700571134686 }, { "clip_ratio/high_max": 0.0007366885984083638, "clip_ratio/high_mean": 0.0007366885984083638, "clip_ratio/low_mean": 0.00010070936405099928, "clip_ratio/low_min": 0.00010070936405099928, "clip_ratio/region_mean": 0.0008373979799216613, "completions/clipped_ratio": 0.0, "completions/max_length": 896.0, "completions/max_terminated_length": 896.0, "completions/mean_length": 608.1199951171875, "completions/mean_terminated_length": 608.1199951171875, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "entropy": 0.1377216622233391, "epoch": 0.16576086956521738, "frac_reward_zero_std": 0.0, "grad_norm": 0.013815459795296192, "learning_rate": 5e-05, "loss": 0.033973757177591324, "num_tokens": 4228531.0, "reward": 0.7030664086341858, "reward_std": 0.04950352758169174, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.2969335913658142, "rewards/length_penalty/std": 0.049503523856401443, "sampling/importance_sampling_ratio/max": 2.5220999717712402, "sampling/importance_sampling_ratio/mean": 0.9949359893798828, "sampling/importance_sampling_ratio/min": 0.05055711790919304, "sampling/sampling_logp_difference/max": 2.984651565551758, "sampling/sampling_logp_difference/mean": 0.019737115129828453, "step": 61, "step_time": 11.692759637953714 }, { "clip_ratio/high_max": 0.0006575177831109613, "clip_ratio/high_mean": 0.0006575177831109613, "clip_ratio/low_mean": 6.264255061978474e-05, "clip_ratio/low_min": 6.264255061978474e-05, "clip_ratio/region_mean": 0.0007201603322755546, "completions/clipped_ratio": 0.0, "completions/max_length": 1995.0, "completions/max_terminated_length": 1995.0, "completions/mean_length": 890.6199951171875, "completions/mean_terminated_length": 890.6199951171875, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.14189326465129853, "epoch": 0.16847826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.015275136567652225, "learning_rate": 5e-05, "loss": 0.03717654570937157, "num_tokens": 4276682.0, "reward": 0.4051269292831421, "reward_std": 0.45795977115631104, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.4348730444908142, "rewards/length_penalty/std": 0.19076396524906158, "sampling/importance_sampling_ratio/max": 2.618985414505005, "sampling/importance_sampling_ratio/mean": 0.9940707683563232, "sampling/importance_sampling_ratio/min": 0.038013607263565063, "sampling/sampling_logp_difference/max": 3.2698111534118652, "sampling/sampling_logp_difference/mean": 0.016751455143094063, "step": 62, "step_time": 23.50743620051071 }, { "clip_ratio/high_max": 0.0008010541961994023, "clip_ratio/high_mean": 0.0008010541961994023, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008010541961994023, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1916.0, "completions/mean_length": 706.97998046875, "completions/mean_terminated_length": 679.6122436523438, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.1526895821094513, "epoch": 0.17119565217391305, "frac_reward_zero_std": 0.0, "grad_norm": 0.01731613650918007, "learning_rate": 5e-05, "loss": 0.04923119395971298, "num_tokens": 4315281.0, "reward": 0.45479491353034973, "reward_std": 0.5850844383239746, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.34520506858825684, "rewards/length_penalty/std": 0.20253480970859528, "sampling/importance_sampling_ratio/max": 2.8465676307678223, "sampling/importance_sampling_ratio/mean": 0.9949560165405273, "sampling/importance_sampling_ratio/min": 0.27247118949890137, "sampling/sampling_logp_difference/max": 1.300222396850586, "sampling/sampling_logp_difference/mean": 0.019272862002253532, "step": 63, "step_time": 23.484382715076208 }, { "clip_ratio/high_max": 0.0003500981023535132, "clip_ratio/high_mean": 0.0003500981023535132, "clip_ratio/low_mean": 5.712283600587398e-05, "clip_ratio/low_min": 5.712283600587398e-05, "clip_ratio/region_mean": 0.00040722094126977026, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 700.2999877929688, "completions/mean_terminated_length": 700.2999877929688, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.13254741728305816, "epoch": 0.17391304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.01813482865691185, "learning_rate": 5e-05, "loss": 0.023750657215714455, "num_tokens": 4353816.0, "reward": 0.6180566549301147, "reward_std": 0.22116827964782715, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.3419433534145355, "rewards/length_penalty/std": 0.09352646768093109, "sampling/importance_sampling_ratio/max": 2.9889914989471436, "sampling/importance_sampling_ratio/mean": 0.9951876401901245, "sampling/importance_sampling_ratio/min": 0.07279559969902039, "sampling/sampling_logp_difference/max": 2.6200997829437256, "sampling/sampling_logp_difference/mean": 0.018573181703686714, "step": 64, "step_time": 13.950854809721932 }, { "clip_ratio/high_max": 0.0010051134566310793, "clip_ratio/high_mean": 0.0010051134566310793, "clip_ratio/low_mean": 7.008167740423233e-05, "clip_ratio/low_min": 7.008167740423233e-05, "clip_ratio/region_mean": 0.0010751951369456947, "completions/clipped_ratio": 0.0, "completions/max_length": 1941.0, "completions/max_terminated_length": 1941.0, "completions/mean_length": 760.8599853515625, "completions/mean_terminated_length": 760.8599853515625, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.15310844779014587, "epoch": 0.1766304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.01888042315840721, "learning_rate": 5e-05, "loss": 0.04399148002266884, "num_tokens": 4395569.0, "reward": 0.5684863328933716, "reward_std": 0.33382996916770935, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979365825653, "rewards/length_penalty/mean": -0.3715136647224426, "rewards/length_penalty/std": 0.16755269467830658, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9941285848617554, "sampling/importance_sampling_ratio/min": 0.07204371690750122, "sampling/sampling_logp_difference/max": 2.6304821968078613, "sampling/sampling_logp_difference/mean": 0.01957661285996437, "step": 65, "step_time": 23.068237875122577 }, { "clip_ratio/high_max": 0.0005175740210688673, "clip_ratio/high_mean": 0.0005175740210688673, "clip_ratio/low_mean": 0.00023222646414069458, "clip_ratio/low_min": 0.00023222646414069458, "clip_ratio/region_mean": 0.0007498004793887958, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 1759.0, "completions/mean_length": 757.97998046875, "completions/mean_terminated_length": 547.9767456054688, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.25827352702617645, "epoch": 0.1793478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.016815872862935066, "learning_rate": 5e-05, "loss": 0.036842163652181625, "num_tokens": 4436518.0, "reward": 0.389892578125, "reward_std": 0.6972166895866394, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.37010741233825684, "rewards/length_penalty/std": 0.28814491629600525, "sampling/importance_sampling_ratio/max": 2.3030846118927, "sampling/importance_sampling_ratio/mean": 0.9900518655776978, "sampling/importance_sampling_ratio/min": 0.16431906819343567, "sampling/sampling_logp_difference/max": 1.8059451580047607, "sampling/sampling_logp_difference/mean": 0.026268478482961655, "step": 66, "step_time": 23.87718580919318 }, { "clip_ratio/high_max": 0.00029754412826150657, "clip_ratio/high_mean": 0.00029754412826150657, "clip_ratio/low_mean": 3.082613984588534e-05, "clip_ratio/low_min": 3.082613984588534e-05, "clip_ratio/region_mean": 0.00032837026519700886, "completions/clipped_ratio": 0.0, "completions/max_length": 1330.0, "completions/max_terminated_length": 1330.0, "completions/mean_length": 671.2000122070312, "completions/mean_terminated_length": 671.2000122070312, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 0.10958393663167953, "epoch": 0.18206521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.014129486866295338, "learning_rate": 5e-05, "loss": 0.03558261692523956, "num_tokens": 4473028.0, "reward": 0.27226561307907104, "reward_std": 0.5440670847892761, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.3277343809604645, "rewards/length_penalty/std": 0.13625548779964447, "sampling/importance_sampling_ratio/max": 2.361050605773926, "sampling/importance_sampling_ratio/mean": 0.996082603931427, "sampling/importance_sampling_ratio/min": 0.0901251807808876, "sampling/sampling_logp_difference/max": 2.406555652618408, "sampling/sampling_logp_difference/mean": 0.016017161309719086, "step": 67, "step_time": 15.848499092971906 }, { "clip_ratio/high_max": 0.0004814961168449372, "clip_ratio/high_mean": 0.0004814961168449372, "clip_ratio/low_mean": 0.00011306432425044477, "clip_ratio/low_min": 0.00011306432425044477, "clip_ratio/region_mean": 0.000594560446916148, "completions/clipped_ratio": 0.0, "completions/max_length": 1327.0, "completions/max_terminated_length": 1327.0, "completions/mean_length": 709.97998046875, "completions/mean_terminated_length": 709.97998046875, "completions/min_length": 377.0, "completions/min_terminated_length": 377.0, "entropy": 0.12251516133546829, "epoch": 0.18478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.017992310225963593, "learning_rate": 5e-05, "loss": 0.034500207751989365, "num_tokens": 4512287.0, "reward": 0.43333005905151367, "reward_std": 0.5297770500183105, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.34666991233825684, "rewards/length_penalty/std": 0.12326952069997787, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9955129027366638, "sampling/importance_sampling_ratio/min": 0.2864378094673157, "sampling/sampling_logp_difference/max": 1.2502338886260986, "sampling/sampling_logp_difference/mean": 0.01752791553735733, "step": 68, "step_time": 16.056356735294685 }, { "clip_ratio/high_max": 0.000786409858847037, "clip_ratio/high_mean": 0.000786409858847037, "clip_ratio/low_mean": 0.00017479720409028233, "clip_ratio/low_min": 0.00017479720409028233, "clip_ratio/region_mean": 0.0009612070745788515, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1685.0, "completions/mean_length": 695.0, "completions/mean_terminated_length": 544.6666870117188, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.2089858502149582, "epoch": 0.1875, "frac_reward_zero_std": 0.0, "grad_norm": 0.01573641039431095, "learning_rate": 5e-05, "loss": 0.06109810620546341, "num_tokens": 4549417.0, "reward": 0.2606445252895355, "reward_std": 0.6902934312820435, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.33935546875, "rewards/length_penalty/std": 0.27389469742774963, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9919227361679077, "sampling/importance_sampling_ratio/min": 0.04992447420954704, "sampling/sampling_logp_difference/max": 2.997243881225586, "sampling/sampling_logp_difference/mean": 0.022705664858222008, "step": 69, "step_time": 24.100224602036178 }, { "clip_ratio/high_max": 0.0003861734687234275, "clip_ratio/high_mean": 0.0003861734687234275, "clip_ratio/low_mean": 8.133393421303481e-05, "clip_ratio/low_min": 8.133393421303481e-05, "clip_ratio/region_mean": 0.00046750739857088776, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1819.0, "completions/mean_length": 974.5999755859375, "completions/mean_terminated_length": 706.25, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.12589940428733826, "epoch": 0.19021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.014950593002140522, "learning_rate": 5e-05, "loss": 0.03755060210824013, "num_tokens": 4601657.0, "reward": -0.27587890625, "reward_std": 0.6061694622039795, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.47587889432907104, "rewards/length_penalty/std": 0.3194302022457123, "sampling/importance_sampling_ratio/max": 2.5902318954467773, "sampling/importance_sampling_ratio/mean": 0.9951319098472595, "sampling/importance_sampling_ratio/min": 0.07221422344446182, "sampling/sampling_logp_difference/max": 2.6281182765960693, "sampling/sampling_logp_difference/mean": 0.015436016023159027, "step": 70, "step_time": 24.753103533759713 }, { "clip_ratio/high_max": 0.0009733674465678632, "clip_ratio/high_mean": 0.0009733674465678632, "clip_ratio/low_mean": 0.00012668214039877058, "clip_ratio/low_min": 0.00012668214039877058, "clip_ratio/region_mean": 0.001100049610249698, "completions/clipped_ratio": 0.0, "completions/max_length": 1306.0, "completions/max_terminated_length": 1306.0, "completions/mean_length": 481.8999938964844, "completions/mean_terminated_length": 481.8999938964844, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.18031663894653321, "epoch": 0.19293478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.017065592110157013, "learning_rate": 5e-05, "loss": 0.02791469730436802, "num_tokens": 4628112.0, "reward": 0.26469725370407104, "reward_std": 0.5313357710838318, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.23530273139476776, "rewards/length_penalty/std": 0.08668895810842514, "sampling/importance_sampling_ratio/max": 2.9714601039886475, "sampling/importance_sampling_ratio/mean": 0.9928731918334961, "sampling/importance_sampling_ratio/min": 0.06918845325708389, "sampling/sampling_logp_difference/max": 2.6709213256835938, "sampling/sampling_logp_difference/mean": 0.026175597682595253, "step": 71, "step_time": 14.497095804661512 }, { "clip_ratio/high_max": 0.000628325465368107, "clip_ratio/high_mean": 0.000628325465368107, "clip_ratio/low_mean": 4.015257873106748e-05, "clip_ratio/low_min": 4.015257873106748e-05, "clip_ratio/region_mean": 0.0006684780470095575, "completions/clipped_ratio": 0.0, "completions/max_length": 1143.0, "completions/max_terminated_length": 1143.0, "completions/mean_length": 483.8999938964844, "completions/mean_terminated_length": 483.8999938964844, "completions/min_length": 178.0, "completions/min_terminated_length": 178.0, "entropy": 0.11079794615507126, "epoch": 0.1956521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.01392416376620531, "learning_rate": 5e-05, "loss": 0.02589317597448826, "num_tokens": 4655977.0, "reward": 0.5037207007408142, "reward_std": 0.5148534774780273, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.23627929389476776, "rewards/length_penalty/std": 0.10259860754013062, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9951469898223877, "sampling/importance_sampling_ratio/min": 0.2387021780014038, "sampling/sampling_logp_difference/max": 1.432538628578186, "sampling/sampling_logp_difference/mean": 0.01793760433793068, "step": 72, "step_time": 13.306627204176039 }, { "clip_ratio/high_max": 0.0005853123526321724, "clip_ratio/high_mean": 0.0005853123526321724, "clip_ratio/low_mean": 0.0001358868816168979, "clip_ratio/low_min": 0.0001358868816168979, "clip_ratio/region_mean": 0.0007211992342490703, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 2014.0, "completions/mean_length": 723.5999755859375, "completions/mean_terminated_length": 608.434814453125, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.17599567174911498, "epoch": 0.1983695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.018252408131957054, "learning_rate": 5e-05, "loss": 0.04385628551244736, "num_tokens": 4694567.0, "reward": 0.22667968273162842, "reward_std": 0.6738052368164062, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.35332030057907104, "rewards/length_penalty/std": 0.26818665862083435, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9931528568267822, "sampling/importance_sampling_ratio/min": 0.20481857657432556, "sampling/sampling_logp_difference/max": 1.5856306552886963, "sampling/sampling_logp_difference/mean": 0.020317068323493004, "step": 73, "step_time": 24.120045095914975 }, { "clip_ratio/high_max": 0.0005197267164476216, "clip_ratio/high_mean": 0.0005197267164476216, "clip_ratio/low_mean": 7.821666076779366e-05, "clip_ratio/low_min": 7.821666076779366e-05, "clip_ratio/region_mean": 0.0005979433772154152, "completions/clipped_ratio": 0.0, "completions/max_length": 916.0, "completions/max_terminated_length": 916.0, "completions/mean_length": 504.55999755859375, "completions/mean_terminated_length": 504.55999755859375, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.14893011450767518, "epoch": 0.20108695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.015567939728498459, "learning_rate": 5e-05, "loss": 0.03300417959690094, "num_tokens": 4723125.0, "reward": 0.15363280475139618, "reward_std": 0.5294827818870544, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.2463671863079071, "rewards/length_penalty/std": 0.09036648273468018, "sampling/importance_sampling_ratio/max": 2.6094131469726562, "sampling/importance_sampling_ratio/mean": 0.9946196675300598, "sampling/importance_sampling_ratio/min": 0.20717699825763702, "sampling/sampling_logp_difference/max": 1.5741817951202393, "sampling/sampling_logp_difference/mean": 0.02010997198522091, "step": 74, "step_time": 11.145676413783804 }, { "clip_ratio/high_max": 0.0009385790792293847, "clip_ratio/high_mean": 0.0009385790792293847, "clip_ratio/low_mean": 0.00018966651987284422, "clip_ratio/low_min": 0.00018966651987284422, "clip_ratio/region_mean": 0.001128245610743761, "completions/clipped_ratio": 0.0, "completions/max_length": 1005.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 429.67999267578125, "completions/mean_terminated_length": 429.67999267578125, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "entropy": 0.1344812273979187, "epoch": 0.20380434782608695, "frac_reward_zero_std": 0.0, "grad_norm": 0.01469445414841175, "learning_rate": 5e-05, "loss": 0.01116997841745615, "num_tokens": 4746939.0, "reward": 0.6301953196525574, "reward_std": 0.38830801844596863, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.2098046839237213, "rewards/length_penalty/std": 0.12929898500442505, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9950018525123596, "sampling/importance_sampling_ratio/min": 0.17992667853832245, "sampling/sampling_logp_difference/max": 1.7152059078216553, "sampling/sampling_logp_difference/mean": 0.022754253819584846, "step": 75, "step_time": 11.691951024346054 }, { "clip_ratio/high_max": 0.00032894672185648235, "clip_ratio/high_mean": 0.00032894672185648235, "clip_ratio/low_mean": 0.0002415040013147518, "clip_ratio/low_min": 0.0002415040013147518, "clip_ratio/region_mean": 0.0005704507231712342, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 2009.0, "completions/mean_length": 591.8200073242188, "completions/mean_terminated_length": 430.0222473144531, "completions/min_length": 148.0, "completions/min_terminated_length": 148.0, "entropy": 0.26585462391376496, "epoch": 0.20652173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.014526180922985077, "learning_rate": 5e-05, "loss": 0.04992569983005524, "num_tokens": 4779870.0, "reward": 0.4910253882408142, "reward_std": 0.6834759712219238, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.2889746129512787, "rewards/length_penalty/std": 0.2923096716403961, "sampling/importance_sampling_ratio/max": 2.685748338699341, "sampling/importance_sampling_ratio/mean": 0.9892884492874146, "sampling/importance_sampling_ratio/min": 0.30007389187812805, "sampling/sampling_logp_difference/max": 1.2037265300750732, "sampling/sampling_logp_difference/mean": 0.0263529010117054, "step": 76, "step_time": 23.11679401784204 }, { "clip_ratio/high_max": 0.0008211379870772361, "clip_ratio/high_mean": 0.0008211379870772361, "clip_ratio/low_mean": 8.669763628859073e-05, "clip_ratio/low_min": 8.669763628859073e-05, "clip_ratio/region_mean": 0.00090783562627621, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1921.0, "completions/mean_length": 535.6799926757812, "completions/mean_terminated_length": 472.66668701171875, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.17671192586421966, "epoch": 0.20923913043478262, "frac_reward_zero_std": 0.0, "grad_norm": 0.01664687879383564, "learning_rate": 5e-05, "loss": 0.04311929643154144, "num_tokens": 4809214.0, "reward": 0.5584374666213989, "reward_std": 0.5280290246009827, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.2615624964237213, "rewards/length_penalty/std": 0.23831947147846222, "sampling/importance_sampling_ratio/max": 2.370129108428955, "sampling/importance_sampling_ratio/mean": 0.9928213953971863, "sampling/importance_sampling_ratio/min": 0.16383084654808044, "sampling/sampling_logp_difference/max": 1.8089207410812378, "sampling/sampling_logp_difference/mean": 0.019636210054159164, "step": 77, "step_time": 22.68614157452248 }, { "clip_ratio/high_max": 0.0005915068177273497, "clip_ratio/high_mean": 0.0005915068177273497, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0005915068177273497, "completions/clipped_ratio": 0.0, "completions/max_length": 1322.0, "completions/max_terminated_length": 1322.0, "completions/mean_length": 444.0199890136719, "completions/mean_terminated_length": 444.0199890136719, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.09074169397354126, "epoch": 0.21195652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.013410658575594425, "learning_rate": 5e-05, "loss": 0.04930553585290909, "num_tokens": 4833715.0, "reward": 0.38319334387779236, "reward_std": 0.5701014995574951, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.21680663526058197, "rewards/length_penalty/std": 0.1205679401755333, "sampling/importance_sampling_ratio/max": 2.962005138397217, "sampling/importance_sampling_ratio/mean": 0.9963091015815735, "sampling/importance_sampling_ratio/min": 0.15177083015441895, "sampling/sampling_logp_difference/max": 1.8853836059570312, "sampling/sampling_logp_difference/mean": 0.014590539038181305, "step": 78, "step_time": 15.235982230165973 }, { "clip_ratio/high_max": 0.0005698791879694909, "clip_ratio/high_mean": 0.0005698791879694909, "clip_ratio/low_mean": 4.1485170368105176e-05, "clip_ratio/low_min": 4.1485170368105176e-05, "clip_ratio/region_mean": 0.0006113643466960639, "completions/clipped_ratio": 0.0, "completions/max_length": 700.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 444.0199890136719, "completions/mean_terminated_length": 444.0199890136719, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.10524996668100357, "epoch": 0.21467391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.013324566185474396, "learning_rate": 5e-05, "loss": 0.010624676942825317, "num_tokens": 4859376.0, "reward": 0.5031933188438416, "reward_std": 0.43716707825660706, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.21680663526058197, "rewards/length_penalty/std": 0.06869058310985565, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9948770403862, "sampling/importance_sampling_ratio/min": 0.3475484549999237, "sampling/sampling_logp_difference/max": 1.1924397945404053, "sampling/sampling_logp_difference/mean": 0.019135504961013794, "step": 79, "step_time": 8.87365493294783 }, { "clip_ratio/high_max": 0.00045147799828555436, "clip_ratio/high_mean": 0.00045147799828555436, "clip_ratio/low_mean": 0.0001845527091063559, "clip_ratio/low_min": 0.0001845527091063559, "clip_ratio/region_mean": 0.0006360307103022933, "completions/clipped_ratio": 0.0, "completions/max_length": 665.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 409.3800048828125, "completions/mean_terminated_length": 409.3800048828125, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 0.11959978342056274, "epoch": 0.21739130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.013123178854584694, "learning_rate": 5e-05, "loss": 0.024036118760704994, "num_tokens": 4883005.0, "reward": 0.2001074105501175, "reward_std": 0.5326245427131653, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.1998925805091858, "rewards/length_penalty/std": 0.05713106691837311, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9956418871879578, "sampling/importance_sampling_ratio/min": 0.3026818335056305, "sampling/sampling_logp_difference/max": 1.1951258182525635, "sampling/sampling_logp_difference/mean": 0.02054966241121292, "step": 80, "step_time": 8.360282597364858 }, { "clip_ratio/high_max": 0.000704864104045555, "clip_ratio/high_mean": 0.000704864104045555, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.000704864104045555, "completions/clipped_ratio": 0.0, "completions/max_length": 818.0, "completions/max_terminated_length": 818.0, "completions/mean_length": 417.5799865722656, "completions/mean_terminated_length": 417.5799865722656, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.1103743776679039, "epoch": 0.22010869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.01447275374084711, "learning_rate": 5e-05, "loss": 0.021041925996541977, "num_tokens": 4908344.0, "reward": 0.556103527545929, "reward_std": 0.4081662595272064, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.20389647781848907, "rewards/length_penalty/std": 0.07568074017763138, "sampling/importance_sampling_ratio/max": 2.083041191101074, "sampling/importance_sampling_ratio/mean": 0.9952544569969177, "sampling/importance_sampling_ratio/min": 0.2227083295583725, "sampling/sampling_logp_difference/max": 1.501892328262329, "sampling/sampling_logp_difference/mean": 0.01730303466320038, "step": 81, "step_time": 10.241758018033579 }, { "clip_ratio/high_max": 0.0006575023173354567, "clip_ratio/high_mean": 0.0006575023173354567, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0006575023173354567, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 294.55999755859375, "completions/mean_terminated_length": 294.55999755859375, "completions/min_length": 119.0, "completions/min_terminated_length": 119.0, "entropy": 0.10942234545946121, "epoch": 0.22282608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.012251164764165878, "learning_rate": 5e-05, "loss": 0.009689167141914368, "num_tokens": 4925922.0, "reward": 0.5361718535423279, "reward_std": 0.5191479921340942, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.4712120592594147, "rewards/length_penalty/mean": -0.1438281238079071, "rewards/length_penalty/std": 0.07002057135105133, "sampling/importance_sampling_ratio/max": 2.1058547496795654, "sampling/importance_sampling_ratio/mean": 0.997086763381958, "sampling/importance_sampling_ratio/min": 0.2412642240524292, "sampling/sampling_logp_difference/max": 1.4218626022338867, "sampling/sampling_logp_difference/mean": 0.018847452476620674, "step": 82, "step_time": 7.392252362798899 }, { "clip_ratio/high_max": 0.00035982736153528094, "clip_ratio/high_mean": 0.00035982736153528094, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00035982736153528094, "completions/clipped_ratio": 0.0, "completions/max_length": 491.0, "completions/max_terminated_length": 491.0, "completions/mean_length": 237.3199920654297, "completions/mean_terminated_length": 237.3199920654297, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "entropy": 0.1137263223528862, "epoch": 0.22554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.012257511727511883, "learning_rate": 5e-05, "loss": 0.013772832229733467, "num_tokens": 4940218.0, "reward": 0.4641210734844208, "reward_std": 0.5065000653266907, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.11587890982627869, "rewards/length_penalty/std": 0.04924225062131882, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9956432580947876, "sampling/importance_sampling_ratio/min": 0.24330933392047882, "sampling/sampling_logp_difference/max": 1.413421630859375, "sampling/sampling_logp_difference/mean": 0.018884913995862007, "step": 83, "step_time": 6.00911526218988 }, { "clip_ratio/high_max": 0.0010932499542832374, "clip_ratio/high_mean": 0.0010932499542832374, "clip_ratio/low_mean": 5.839415825903416e-05, "clip_ratio/low_min": 5.839415825903416e-05, "clip_ratio/region_mean": 0.0011516441009007394, "completions/clipped_ratio": 0.0, "completions/max_length": 1617.0, "completions/max_terminated_length": 1617.0, "completions/mean_length": 375.1600036621094, "completions/mean_terminated_length": 375.1600036621094, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "entropy": 0.1211256891489029, "epoch": 0.22826086956521738, "frac_reward_zero_std": 0.0, "grad_norm": 0.019089948385953903, "learning_rate": 5e-05, "loss": 0.03538642078638077, "num_tokens": 4961956.0, "reward": 0.5568163990974426, "reward_std": 0.4542848467826843, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.18318359553813934, "rewards/length_penalty/std": 0.18591263890266418, "sampling/importance_sampling_ratio/max": 2.340315580368042, "sampling/importance_sampling_ratio/mean": 0.995381236076355, "sampling/importance_sampling_ratio/min": 0.24651946127414703, "sampling/sampling_logp_difference/max": 1.4003143310546875, "sampling/sampling_logp_difference/mean": 0.018075764179229736, "step": 84, "step_time": 18.067105296999216 }, { "clip_ratio/high_max": 0.0010971381212584673, "clip_ratio/high_mean": 0.0010971381212584673, "clip_ratio/low_mean": 9.907572530210018e-05, "clip_ratio/low_min": 9.907572530210018e-05, "clip_ratio/region_mean": 0.0011962138232775033, "completions/clipped_ratio": 0.0, "completions/max_length": 1556.0, "completions/max_terminated_length": 1556.0, "completions/mean_length": 362.79998779296875, "completions/mean_terminated_length": 362.79998779296875, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 0.2110247790813446, "epoch": 0.23097826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.021158233284950256, "learning_rate": 5e-05, "loss": 0.044072747230529785, "num_tokens": 4982116.0, "reward": 0.3828515410423279, "reward_std": 0.5390759110450745, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.17714843153953552, "rewards/length_penalty/std": 0.12832064926624298, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9906708598136902, "sampling/importance_sampling_ratio/min": 0.171915665268898, "sampling/sampling_logp_difference/max": 1.7607512474060059, "sampling/sampling_logp_difference/mean": 0.028823882341384888, "step": 85, "step_time": 16.47355701122433 }, { "clip_ratio/high_max": 0.0005582125799264759, "clip_ratio/high_mean": 0.0005582125799264759, "clip_ratio/low_mean": 0.00011848341673612595, "clip_ratio/low_min": 0.00011848341673612595, "clip_ratio/region_mean": 0.0006766959966626018, "completions/clipped_ratio": 0.0, "completions/max_length": 332.0, "completions/max_terminated_length": 332.0, "completions/mean_length": 183.47999572753906, "completions/mean_terminated_length": 183.47999572753906, "completions/min_length": 57.0, "completions/min_terminated_length": 57.0, "entropy": 0.14162901043891907, "epoch": 0.23369565217391305, "frac_reward_zero_std": 0.0, "grad_norm": 0.01095115952193737, "learning_rate": 5e-05, "loss": 0.007650045212358236, "num_tokens": 4993390.0, "reward": 0.8504101634025574, "reward_std": 0.24047701060771942, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979514837265, "rewards/length_penalty/mean": -0.08958984166383743, "rewards/length_penalty/std": 0.0338219590485096, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9944058656692505, "sampling/importance_sampling_ratio/min": 0.1472282111644745, "sampling/sampling_logp_difference/max": 1.915771484375, "sampling/sampling_logp_difference/mean": 0.02622547745704651, "step": 86, "step_time": 4.271973532857373 }, { "clip_ratio/high_max": 0.0005829900386743247, "clip_ratio/high_mean": 0.0005829900386743247, "clip_ratio/low_mean": 0.0003810461843386292, "clip_ratio/low_min": 0.0003810461843386292, "clip_ratio/region_mean": 0.0009640362113714218, "completions/clipped_ratio": 0.0, "completions/max_length": 630.0, "completions/max_terminated_length": 630.0, "completions/mean_length": 281.739990234375, "completions/mean_terminated_length": 281.739990234375, "completions/min_length": 74.0, "completions/min_terminated_length": 74.0, "entropy": 0.10393131375312806, "epoch": 0.23641304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.013417479582130909, "learning_rate": 5e-05, "loss": 0.014699541963636875, "num_tokens": 5009687.0, "reward": 0.422431617975235, "reward_std": 0.5398931503295898, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.13756835460662842, "rewards/length_penalty/std": 0.07445195317268372, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9958858489990234, "sampling/importance_sampling_ratio/min": 0.13931959867477417, "sampling/sampling_logp_difference/max": 1.970984697341919, "sampling/sampling_logp_difference/mean": 0.018261417746543884, "step": 87, "step_time": 7.5571192188654095 }, { "clip_ratio/high_max": 0.0015571706928312779, "clip_ratio/high_mean": 0.0015571706928312779, "clip_ratio/low_mean": 0.0001285347039811313, "clip_ratio/low_min": 0.0001285347039811313, "clip_ratio/region_mean": 0.0016857054084539413, "completions/clipped_ratio": 0.0, "completions/max_length": 400.0, "completions/max_terminated_length": 400.0, "completions/mean_length": 182.1599884033203, "completions/mean_terminated_length": 182.1599884033203, "completions/min_length": 41.0, "completions/min_terminated_length": 41.0, "entropy": 0.15183552503585815, "epoch": 0.2391304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.011707360856235027, "learning_rate": 5e-05, "loss": 0.0038167578168213367, "num_tokens": 5021685.0, "reward": 0.6910547018051147, "reward_std": 0.4340381622314453, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.08894531428813934, "rewards/length_penalty/std": 0.052339497953653336, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9942026138305664, "sampling/importance_sampling_ratio/min": 0.1707821637392044, "sampling/sampling_logp_difference/max": 1.7673664093017578, "sampling/sampling_logp_difference/mean": 0.02865365333855152, "step": 88, "step_time": 5.0701005139853805 }, { "clip_ratio/high_max": 0.0011892693408299237, "clip_ratio/high_mean": 0.0011892693408299237, "clip_ratio/low_mean": 0.000522052450105548, "clip_ratio/low_min": 0.000522052450105548, "clip_ratio/region_mean": 0.0017113217909354717, "completions/clipped_ratio": 0.0, "completions/max_length": 512.0, "completions/max_terminated_length": 512.0, "completions/mean_length": 213.0, "completions/mean_terminated_length": 213.0, "completions/min_length": 63.0, "completions/min_terminated_length": 63.0, "entropy": 0.15271458625793458, "epoch": 0.2418478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.010374904610216618, "learning_rate": 5e-05, "loss": 0.011677139438688755, "num_tokens": 5035795.0, "reward": 0.3359960913658142, "reward_std": 0.5273035168647766, "rewards/correctness/mean": 0.4399999976158142, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.10400390625, "rewards/length_penalty/std": 0.061812058091163635, "sampling/importance_sampling_ratio/max": 2.3226864337921143, "sampling/importance_sampling_ratio/mean": 0.993018627166748, "sampling/importance_sampling_ratio/min": 0.08702941983938217, "sampling/sampling_logp_difference/max": 2.4415090084075928, "sampling/sampling_logp_difference/mean": 0.02671627141535282, "step": 89, "step_time": 6.264251670800149 }, { "clip_ratio/high_max": 0.0012449576752260327, "clip_ratio/high_mean": 0.0012449576752260327, "clip_ratio/low_mean": 0.00039986594929359854, "clip_ratio/low_min": 0.00039986594929359854, "clip_ratio/region_mean": 0.001644823612878099, "completions/clipped_ratio": 0.0, "completions/max_length": 1107.0, "completions/max_terminated_length": 1107.0, "completions/mean_length": 258.67999267578125, "completions/mean_terminated_length": 258.67999267578125, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "entropy": 0.17646301686763763, "epoch": 0.24456521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.013736448250710964, "learning_rate": 5e-05, "loss": 0.030243348330259323, "num_tokens": 5051759.0, "reward": 0.53369140625, "reward_std": 0.510785698890686, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.47851815819740295, "rewards/length_penalty/mean": -0.1263085901737213, "rewards/length_penalty/std": 0.08943063020706177, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9928296804428101, "sampling/importance_sampling_ratio/min": 0.09859311580657959, "sampling/sampling_logp_difference/max": 2.31675386428833, "sampling/sampling_logp_difference/mean": 0.028476905077695847, "step": 90, "step_time": 11.927960007218644 }, { "clip_ratio/high_max": 0.0011836130055598915, "clip_ratio/high_mean": 0.0011836130055598915, "clip_ratio/low_mean": 0.0006012129248119891, "clip_ratio/low_min": 0.0006012129248119891, "clip_ratio/region_mean": 0.0017848259536549448, "completions/clipped_ratio": 0.0, "completions/max_length": 784.0, "completions/max_terminated_length": 784.0, "completions/mean_length": 168.05999755859375, "completions/mean_terminated_length": 168.05999755859375, "completions/min_length": 43.0, "completions/min_terminated_length": 43.0, "entropy": 0.1712634652853012, "epoch": 0.24728260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.011951963417232037, "learning_rate": 5e-05, "loss": 0.021161450073122978, "num_tokens": 5065332.0, "reward": 0.5579394698143005, "reward_std": 0.5229561924934387, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.0820605456829071, "rewards/length_penalty/std": 0.08735550940036774, "sampling/importance_sampling_ratio/max": 2.185065507888794, "sampling/importance_sampling_ratio/mean": 0.9940653443336487, "sampling/importance_sampling_ratio/min": 0.12851713597774506, "sampling/sampling_logp_difference/max": 2.0516929626464844, "sampling/sampling_logp_difference/mean": 0.02416444756090641, "step": 91, "step_time": 9.349531583720818 }, { "clip_ratio/high_max": 0.0010452893213368953, "clip_ratio/high_mean": 0.0010452893213368953, "clip_ratio/low_mean": 0.00012650220887735485, "clip_ratio/low_min": 0.00012650220887735485, "clip_ratio/region_mean": 0.00117179153021425, "completions/clipped_ratio": 0.0, "completions/max_length": 292.0, "completions/max_terminated_length": 292.0, "completions/mean_length": 136.17999267578125, "completions/mean_terminated_length": 136.17999267578125, "completions/min_length": 51.0, "completions/min_terminated_length": 51.0, "entropy": 0.15401647984981537, "epoch": 0.25, "frac_reward_zero_std": 0.0, "grad_norm": 0.015128343366086483, "learning_rate": 5e-05, "loss": 0.008885608986020088, "num_tokens": 5074531.0, "reward": 0.5135058760643005, "reward_std": 0.5054919719696045, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.06649413704872131, "rewards/length_penalty/std": 0.029116734862327576, "sampling/importance_sampling_ratio/max": 2.3821372985839844, "sampling/importance_sampling_ratio/mean": 0.9937571287155151, "sampling/importance_sampling_ratio/min": 0.09059903770685196, "sampling/sampling_logp_difference/max": 2.4013116359710693, "sampling/sampling_logp_difference/mean": 0.024070926010608673, "step": 92, "step_time": 4.327101199189201 }, { "clip_ratio/high_max": 0.0013822196982800961, "clip_ratio/high_mean": 0.0013822196982800961, "clip_ratio/low_mean": 0.0005461856024339795, "clip_ratio/low_min": 0.0005461856024339795, "clip_ratio/region_mean": 0.0019284053007140756, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1163.0, "completions/mean_length": 264.32000732421875, "completions/mean_terminated_length": 190.0, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "entropy": 0.2661470055580139, "epoch": 0.25271739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.020688654854893684, "learning_rate": 5e-05, "loss": 0.10969983786344528, "num_tokens": 5093367.0, "reward": 0.5709375143051147, "reward_std": 0.6100456118583679, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.1290625035762787, "rewards/length_penalty/std": 0.21535643935203552, "sampling/importance_sampling_ratio/max": 2.3409512042999268, "sampling/importance_sampling_ratio/mean": 0.990288496017456, "sampling/importance_sampling_ratio/min": 0.213004007935524, "sampling/sampling_logp_difference/max": 1.546444296836853, "sampling/sampling_logp_difference/mean": 0.02781936153769493, "step": 93, "step_time": 22.407282277243212 }, { "clip_ratio/high_max": 0.0015437419177033007, "clip_ratio/high_mean": 0.0015437419177033007, "clip_ratio/low_mean": 0.00012154360301792621, "clip_ratio/low_min": 0.00012154360301792621, "clip_ratio/region_mean": 0.0016652855207212268, "completions/clipped_ratio": 0.0, "completions/max_length": 1413.0, "completions/max_terminated_length": 1413.0, "completions/mean_length": 220.3199920654297, "completions/mean_terminated_length": 220.3199920654297, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 0.22524437308311462, "epoch": 0.2554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.020771974697709084, "learning_rate": 5e-05, "loss": 0.052876319736242294, "num_tokens": 5110413.0, "reward": 0.412421852350235, "reward_std": 0.5003185272216797, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.10757812857627869, "rewards/length_penalty/std": 0.1135673075914383, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9909684062004089, "sampling/importance_sampling_ratio/min": 0.32960575819015503, "sampling/sampling_logp_difference/max": 1.5696799755096436, "sampling/sampling_logp_difference/mean": 0.028162555769085884, "step": 94, "step_time": 15.704104893608019 }, { "clip_ratio/high_max": 0.001439261925406754, "clip_ratio/high_mean": 0.001439261925406754, "clip_ratio/low_mean": 0.00019417476141825318, "clip_ratio/low_min": 0.00019417476141825318, "clip_ratio/region_mean": 0.0016334366519004107, "completions/clipped_ratio": 0.0, "completions/max_length": 194.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 97.45999908447266, "completions/mean_terminated_length": 97.45999908447266, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.25395846366882324, "epoch": 0.25815217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.011895176954567432, "learning_rate": 5e-05, "loss": 0.002930461196228862, "num_tokens": 5117656.0, "reward": 0.6524121165275574, "reward_std": 0.46509990096092224, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.047587890177965164, "rewards/length_penalty/std": 0.019898543134331703, "sampling/importance_sampling_ratio/max": 2.9863996505737305, "sampling/importance_sampling_ratio/mean": 0.9909448623657227, "sampling/importance_sampling_ratio/min": 0.4123409688472748, "sampling/sampling_logp_difference/max": 1.0940685272216797, "sampling/sampling_logp_difference/mean": 0.03773015737533569, "step": 95, "step_time": 2.79798654303886 }, { "clip_ratio/high_max": 0.0014799370197579264, "clip_ratio/high_mean": 0.0014799370197579264, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014799370197579264, "completions/clipped_ratio": 0.0, "completions/max_length": 551.0, "completions/max_terminated_length": 551.0, "completions/mean_length": 97.69999694824219, "completions/mean_terminated_length": 97.69999694824219, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.20367034077644347, "epoch": 0.2608695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.013027592562139034, "learning_rate": 5e-05, "loss": 0.01779348962008953, "num_tokens": 5126071.0, "reward": 0.552294909954071, "reward_std": 0.5121820569038391, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.04770507663488388, "rewards/length_penalty/std": 0.042042821645736694, "sampling/importance_sampling_ratio/max": 2.441518783569336, "sampling/importance_sampling_ratio/mean": 0.991175651550293, "sampling/importance_sampling_ratio/min": 0.06918129324913025, "sampling/sampling_logp_difference/max": 2.671024799346924, "sampling/sampling_logp_difference/mean": 0.03059057705104351, "step": 96, "step_time": 6.202032637782395 }, { "clip_ratio/high_max": 0.0011401220224797725, "clip_ratio/high_mean": 0.0011401220224797725, "clip_ratio/low_mean": 0.0009177979780361056, "clip_ratio/low_min": 0.0009177979780361056, "clip_ratio/region_mean": 0.002057919977232814, "completions/clipped_ratio": 0.0, "completions/max_length": 224.0, "completions/max_terminated_length": 224.0, "completions/mean_length": 86.97999572753906, "completions/mean_terminated_length": 86.97999572753906, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 0.2265522986650467, "epoch": 0.26358695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.01077208761125803, "learning_rate": 5e-05, "loss": 0.0009587033418938518, "num_tokens": 5133350.0, "reward": 0.4375292956829071, "reward_std": 0.5013757944107056, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.04247070476412773, "rewards/length_penalty/std": 0.021450260654091835, "sampling/importance_sampling_ratio/max": 2.6412837505340576, "sampling/importance_sampling_ratio/mean": 0.9908226728439331, "sampling/importance_sampling_ratio/min": 0.3037787675857544, "sampling/sampling_logp_difference/max": 1.191455602645874, "sampling/sampling_logp_difference/mean": 0.033219028264284134, "step": 97, "step_time": 3.065898902947083 }, { "clip_ratio/high_max": 0.0010683529078960418, "clip_ratio/high_mean": 0.0010683529078960418, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010683529078960418, "completions/clipped_ratio": 0.0, "completions/max_length": 150.0, "completions/max_terminated_length": 150.0, "completions/mean_length": 55.55999755859375, "completions/mean_terminated_length": 55.55999755859375, "completions/min_length": 18.0, "completions/min_terminated_length": 18.0, "entropy": 0.2746728152036667, "epoch": 0.266304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.012212988920509815, "learning_rate": 5e-05, "loss": 0.007884243503212929, "num_tokens": 5138268.0, "reward": 0.372871071100235, "reward_std": 0.5017322301864624, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.027128906920552254, "rewards/length_penalty/std": 0.014021570794284344, "sampling/importance_sampling_ratio/max": 2.170682907104492, "sampling/importance_sampling_ratio/mean": 0.9889690279960632, "sampling/importance_sampling_ratio/min": 0.35793039202690125, "sampling/sampling_logp_difference/max": 1.0274168252944946, "sampling/sampling_logp_difference/mean": 0.042866162955760956, "step": 98, "step_time": 2.4067599647678435 }, { "clip_ratio/high_max": 0.0008171118563041091, "clip_ratio/high_mean": 0.0008171118563041091, "clip_ratio/low_mean": 0.0007390069542452693, "clip_ratio/low_min": 0.0007390069542452693, "clip_ratio/region_mean": 0.0015561188105493785, "completions/clipped_ratio": 0.0, "completions/max_length": 94.0, "completions/max_terminated_length": 94.0, "completions/mean_length": 50.91999816894531, "completions/mean_terminated_length": 50.91999816894531, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 0.21310125291347504, "epoch": 0.26902173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.011825033463537693, "learning_rate": 5e-05, "loss": 0.0036260976921766996, "num_tokens": 5143724.0, "reward": 0.7351366877555847, "reward_std": 0.4356699287891388, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.02486328035593033, "rewards/length_penalty/std": 0.008906659670174122, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9978911876678467, "sampling/importance_sampling_ratio/min": 0.3821065127849579, "sampling/sampling_logp_difference/max": 1.398585319519043, "sampling/sampling_logp_difference/mean": 0.037688616663217545, "step": 99, "step_time": 2.0239664369728416 }, { "clip_ratio/high_max": 0.0019492614082992077, "clip_ratio/high_mean": 0.0019492614082992077, "clip_ratio/low_mean": 0.0007079645991325378, "clip_ratio/low_min": 0.0007079645991325378, "clip_ratio/region_mean": 0.0026572260074317457, "completions/clipped_ratio": 0.0, "completions/max_length": 125.0, "completions/max_terminated_length": 125.0, "completions/mean_length": 52.87999725341797, "completions/mean_terminated_length": 52.87999725341797, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 0.22497422993183136, "epoch": 0.2717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.01131928339600563, "learning_rate": 5e-05, "loss": 0.004754737485200167, "num_tokens": 5148598.0, "reward": 0.7741796970367432, "reward_std": 0.4040992259979248, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.025820313021540642, "rewards/length_penalty/std": 0.011249576695263386, "sampling/importance_sampling_ratio/max": 2.4267683029174805, "sampling/importance_sampling_ratio/mean": 0.9914778470993042, "sampling/importance_sampling_ratio/min": 0.27528116106987, "sampling/sampling_logp_difference/max": 1.2899622917175293, "sampling/sampling_logp_difference/mean": 0.03707539662718773, "step": 100, "step_time": 2.2108143279328942 }, { "clip_ratio/high_max": 0.0009703139308840037, "clip_ratio/high_mean": 0.0009703139308840037, "clip_ratio/low_mean": 0.0008246284909546375, "clip_ratio/low_min": 0.0008246284909546375, "clip_ratio/region_mean": 0.0017949424218386412, "completions/clipped_ratio": 0.0, "completions/max_length": 244.0, "completions/max_terminated_length": 244.0, "completions/mean_length": 53.099998474121094, "completions/mean_terminated_length": 53.099998474121094, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 0.21071316599845885, "epoch": 0.27445652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.009538685902953148, "learning_rate": 5e-05, "loss": 0.0037817161064594984, "num_tokens": 5154143.0, "reward": 0.47407224774360657, "reward_std": 0.5069644451141357, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.02592773362994194, "rewards/length_penalty/std": 0.01806822419166565, "sampling/importance_sampling_ratio/max": 2.1848325729370117, "sampling/importance_sampling_ratio/mean": 0.9923020005226135, "sampling/importance_sampling_ratio/min": 0.4618144631385803, "sampling/sampling_logp_difference/max": 0.7815392017364502, "sampling/sampling_logp_difference/mean": 0.032437488436698914, "step": 101, "step_time": 3.6693791849538684 }, { "clip_ratio/high_max": 0.0012555033899843693, "clip_ratio/high_mean": 0.0012555033899843693, "clip_ratio/low_mean": 0.0008362280670553446, "clip_ratio/low_min": 0.0008362280670553446, "clip_ratio/region_mean": 0.0020917314570397137, "completions/clipped_ratio": 0.0, "completions/max_length": 198.0, "completions/max_terminated_length": 198.0, "completions/mean_length": 49.15999984741211, "completions/mean_terminated_length": 49.15999984741211, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.27187140583992003, "epoch": 0.27717391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.010691394098103046, "learning_rate": 5e-05, "loss": 0.004616108722984791, "num_tokens": 5159531.0, "reward": 0.5159960985183716, "reward_std": 0.5072278380393982, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.024003906175494194, "rewards/length_penalty/std": 0.018878955394029617, "sampling/importance_sampling_ratio/max": 1.9397785663604736, "sampling/importance_sampling_ratio/mean": 0.9903997182846069, "sampling/importance_sampling_ratio/min": 0.20729371905326843, "sampling/sampling_logp_difference/max": 1.5736185312271118, "sampling/sampling_logp_difference/mean": 0.03977171704173088, "step": 102, "step_time": 2.7362761148251593 }, { "clip_ratio/high_max": 0.000531679327832535, "clip_ratio/high_mean": 0.000531679327832535, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.000531679327832535, "completions/clipped_ratio": 0.0, "completions/max_length": 1734.0, "completions/max_terminated_length": 1734.0, "completions/mean_length": 133.86000061035156, "completions/mean_terminated_length": 133.86000061035156, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.40329901576042176, "epoch": 0.2798913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.01844567432999611, "learning_rate": 5e-05, "loss": 0.03575665503740311, "num_tokens": 5170464.0, "reward": 0.27463865280151367, "reward_std": 0.5309969782829285, "rewards/correctness/mean": 0.3400000035762787, "rewards/correctness/std": 0.4785180985927582, "rewards/length_penalty/mean": -0.06536132842302322, "rewards/length_penalty/std": 0.16204895079135895, "sampling/importance_sampling_ratio/max": 2.119347333908081, "sampling/importance_sampling_ratio/mean": 0.9828487634658813, "sampling/importance_sampling_ratio/min": 0.37476417422294617, "sampling/sampling_logp_difference/max": 0.981458306312561, "sampling/sampling_logp_difference/mean": 0.039075400680303574, "step": 103, "step_time": 18.458227863069624 }, { "clip_ratio/high_max": 0.002266806270927191, "clip_ratio/high_mean": 0.002266806270927191, "clip_ratio/low_mean": 0.00035335689317435027, "clip_ratio/low_min": 0.00035335689317435027, "clip_ratio/region_mean": 0.0026201631408184767, "completions/clipped_ratio": 0.0, "completions/max_length": 222.0, "completions/max_terminated_length": 222.0, "completions/mean_length": 51.55999755859375, "completions/mean_terminated_length": 51.55999755859375, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "entropy": 0.33404588103294375, "epoch": 0.2826086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.013699734583497047, "learning_rate": 5e-05, "loss": 0.011154095642268658, "num_tokens": 5176552.0, "reward": 0.03482421860098839, "reward_std": 0.23582740128040314, "rewards/correctness/mean": 0.05999999865889549, "rewards/correctness/std": 0.2398979514837265, "rewards/length_penalty/mean": -0.025175781920552254, "rewards/length_penalty/std": 0.024918686598539352, "sampling/importance_sampling_ratio/max": 2.478984832763672, "sampling/importance_sampling_ratio/mean": 0.9874392747879028, "sampling/importance_sampling_ratio/min": 0.20078380405902863, "sampling/sampling_logp_difference/max": 1.6055265665054321, "sampling/sampling_logp_difference/mean": 0.04058130085468292, "step": 104, "step_time": 2.9577652697917074 }, { "clip_ratio/high_max": 0.0016553595196455717, "clip_ratio/high_mean": 0.0016553595196455717, "clip_ratio/low_mean": 0.0009756097570061684, "clip_ratio/low_min": 0.0009756097570061684, "clip_ratio/region_mean": 0.00263096927665174, "completions/clipped_ratio": 0.0, "completions/max_length": 65.0, "completions/max_terminated_length": 65.0, "completions/mean_length": 23.059999465942383, "completions/mean_terminated_length": 23.059999465942383, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.21856221556663513, "epoch": 0.28532608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.012349529191851616, "learning_rate": 5e-05, "loss": 0.00167317152954638, "num_tokens": 5180065.0, "reward": 0.3287402391433716, "reward_std": 0.4786170721054077, "rewards/correctness/mean": 0.3400000035762787, "rewards/correctness/std": 0.4785180985927582, "rewards/length_penalty/mean": -0.011259765364229679, "rewards/length_penalty/std": 0.005874643102288246, "sampling/importance_sampling_ratio/max": 1.7306820154190063, "sampling/importance_sampling_ratio/mean": 0.9920327067375183, "sampling/importance_sampling_ratio/min": 0.28651663661003113, "sampling/sampling_logp_difference/max": 1.2499587535858154, "sampling/sampling_logp_difference/mean": 0.03913699463009834, "step": 105, "step_time": 1.7049150760285556 }, { "clip_ratio/high_max": 0.0011111111380159855, "clip_ratio/high_mean": 0.0011111111380159855, "clip_ratio/low_mean": 0.0008888889104127883, "clip_ratio/low_min": 0.0008888889104127883, "clip_ratio/region_mean": 0.002000000048428774, "completions/clipped_ratio": 0.0, "completions/max_length": 67.0, "completions/max_terminated_length": 67.0, "completions/mean_length": 19.81999969482422, "completions/mean_terminated_length": 19.81999969482422, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.20343194901943207, "epoch": 0.28804347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.009475941769778728, "learning_rate": 5e-05, "loss": 0.000543340458534658, "num_tokens": 5183536.0, "reward": 0.37032225728034973, "reward_std": 0.48816052079200745, "rewards/correctness/mean": 0.3799999952316284, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.009677734225988388, "rewards/length_penalty/std": 0.005778003018349409, "sampling/importance_sampling_ratio/max": 2.096174955368042, "sampling/importance_sampling_ratio/mean": 0.9992754459381104, "sampling/importance_sampling_ratio/min": 0.2721841633319855, "sampling/sampling_logp_difference/max": 1.301276445388794, "sampling/sampling_logp_difference/mean": 0.03429510071873665, "step": 106, "step_time": 1.7274974128231406 }, { "clip_ratio/high_max": 0.0036198318470269442, "clip_ratio/high_mean": 0.0036198318470269442, "clip_ratio/low_mean": 0.0018731069285422564, "clip_ratio/low_min": 0.0018731069285422564, "clip_ratio/region_mean": 0.0054929387755692, "completions/clipped_ratio": 0.0, "completions/max_length": 137.0, "completions/max_terminated_length": 137.0, "completions/mean_length": 19.479999542236328, "completions/mean_terminated_length": 19.479999542236328, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.27643795013427735, "epoch": 0.2907608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.014136921614408493, "learning_rate": 5e-05, "loss": 0.0054918513633310795, "num_tokens": 5188550.0, "reward": 0.1904882788658142, "reward_std": 0.4065392017364502, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.009511718526482582, "rewards/length_penalty/std": 0.00997720006853342, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.993407130241394, "sampling/importance_sampling_ratio/min": 0.28779035806655884, "sampling/sampling_logp_difference/max": 1.2455229759216309, "sampling/sampling_logp_difference/mean": 0.040501918643713, "step": 107, "step_time": 2.2549472115933895 }, { "clip_ratio/high_max": 0.0005633802618831396, "clip_ratio/high_mean": 0.0005633802618831396, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0005633802618831396, "completions/clipped_ratio": 0.0, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 26.899999618530273, "completions/mean_terminated_length": 26.899999618530273, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.2395658552646637, "epoch": 0.29347826086956524, "frac_reward_zero_std": 0.0, "grad_norm": 0.013888458721339703, "learning_rate": 5e-05, "loss": 0.004586397670209408, "num_tokens": 5192945.0, "reward": 0.3868652284145355, "reward_std": 0.5013245940208435, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.013134765438735485, "rewards/length_penalty/std": 0.009524103254079819, "sampling/importance_sampling_ratio/max": 1.663638710975647, "sampling/importance_sampling_ratio/mean": 0.9929983615875244, "sampling/importance_sampling_ratio/min": 0.4642603099346161, "sampling/sampling_logp_difference/max": 0.7673099040985107, "sampling/sampling_logp_difference/mean": 0.03226347267627716, "step": 108, "step_time": 1.7853918452747166 }, { "clip_ratio/high_max": 0.0024331796914339064, "clip_ratio/high_mean": 0.0024331796914339064, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024331796914339064, "completions/clipped_ratio": 0.0, "completions/max_length": 43.0, "completions/max_terminated_length": 43.0, "completions/mean_length": 15.059999465942383, "completions/mean_terminated_length": 15.059999465942383, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.2164798602461815, "epoch": 0.296195652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.010043303482234478, "learning_rate": 5e-05, "loss": 0.00015821788110770285, "num_tokens": 5196038.0, "reward": 0.512646496295929, "reward_std": 0.5045772790908813, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.007353515829890966, "rewards/length_penalty/std": 0.0035750912502408028, "sampling/importance_sampling_ratio/max": 1.714004397392273, "sampling/importance_sampling_ratio/mean": 0.9908314347267151, "sampling/importance_sampling_ratio/min": 0.2964578866958618, "sampling/sampling_logp_difference/max": 1.2158501148223877, "sampling/sampling_logp_difference/mean": 0.03589223325252533, "step": 109, "step_time": 1.5638836093712598 }, { "clip_ratio/high_max": 0.002983189094811678, "clip_ratio/high_mean": 0.002983189094811678, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002983189094811678, "completions/clipped_ratio": 0.0, "completions/max_length": 59.0, "completions/max_terminated_length": 59.0, "completions/mean_length": 18.34000015258789, "completions/mean_terminated_length": 18.34000015258789, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.2114427149295807, "epoch": 0.29891304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.013578943908214569, "learning_rate": 5e-05, "loss": 7.86829914432019e-05, "num_tokens": 5199535.0, "reward": 0.3510449230670929, "reward_std": 0.48104938864707947, "rewards/correctness/mean": 0.36000001430511475, "rewards/correctness/std": 0.4848732054233551, "rewards/length_penalty/mean": -0.008955078199505806, "rewards/length_penalty/std": 0.007135857827961445, "sampling/importance_sampling_ratio/max": 1.491443157196045, "sampling/importance_sampling_ratio/mean": 0.9869861006736755, "sampling/importance_sampling_ratio/min": 0.17444418370723724, "sampling/sampling_logp_difference/max": 1.7461504936218262, "sampling/sampling_logp_difference/mean": 0.037049755454063416, "step": 110, "step_time": 1.6775788262020797 }, { "clip_ratio/high_max": 0.006576033774763346, "clip_ratio/high_mean": 0.006576033774763346, "clip_ratio/low_mean": 0.0028456510975956918, "clip_ratio/low_min": 0.0028456510975956918, "clip_ratio/region_mean": 0.009421684872359037, "completions/clipped_ratio": 0.0, "completions/max_length": 44.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 14.679999351501465, "completions/mean_terminated_length": 14.679999351501465, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.12253115922212601, "epoch": 0.3016304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.011963631957769394, "learning_rate": 5e-05, "loss": -0.0008805968682281673, "num_tokens": 5202929.0, "reward": 0.5328320264816284, "reward_std": 0.5013919472694397, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.007167968899011612, "rewards/length_penalty/std": 0.005465332418680191, "sampling/importance_sampling_ratio/max": 2.8982994556427, "sampling/importance_sampling_ratio/mean": 0.9911866188049316, "sampling/importance_sampling_ratio/min": 0.09863191097974777, "sampling/sampling_logp_difference/max": 2.3163604736328125, "sampling/sampling_logp_difference/mean": 0.04423119127750397, "step": 111, "step_time": 1.568905862979591 }, { "clip_ratio/high_max": 0.007079645991325378, "clip_ratio/high_mean": 0.007079645991325378, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007079645991325378, "completions/clipped_ratio": 0.0, "completions/max_length": 34.0, "completions/max_terminated_length": 34.0, "completions/mean_length": 12.84000015258789, "completions/mean_terminated_length": 12.84000015258789, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.15926921963691712, "epoch": 0.30434782608695654, "frac_reward_zero_std": 0.0, "grad_norm": 0.025274818763136864, "learning_rate": 5e-05, "loss": 0.0012229093117639422, "num_tokens": 5205871.0, "reward": 0.1937304586172104, "reward_std": 0.40328681468963623, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.006269531324505806, "rewards/length_penalty/std": 0.0021732966415584087, "sampling/importance_sampling_ratio/max": 1.719705581665039, "sampling/importance_sampling_ratio/mean": 0.9994849562644958, "sampling/importance_sampling_ratio/min": 0.5315423607826233, "sampling/sampling_logp_difference/max": 0.6319724321365356, "sampling/sampling_logp_difference/mean": 0.016026850789785385, "step": 112, "step_time": 1.4825945999473333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0013245033100247384, "clip_ratio/low_min": 0.0013245033100247384, "clip_ratio/region_mean": 0.0013245033100247384, "completions/clipped_ratio": 0.0, "completions/max_length": 28.0, "completions/max_terminated_length": 28.0, "completions/mean_length": 15.819999694824219, "completions/mean_terminated_length": 15.819999694824219, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.13411899358034135, "epoch": 0.3070652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.013565887697041035, "learning_rate": 5e-05, "loss": 0.00022481636551674455, "num_tokens": 5209972.0, "reward": 0.1722753942012787, "reward_std": 0.38941752910614014, "rewards/correctness/mean": 0.18000000715255737, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.007724609225988388, "rewards/length_penalty/std": 0.0031237073708325624, "sampling/importance_sampling_ratio/max": 1.3955680131912231, "sampling/importance_sampling_ratio/mean": 0.9914500117301941, "sampling/importance_sampling_ratio/min": 0.0927952229976654, "sampling/sampling_logp_difference/max": 2.3773601055145264, "sampling/sampling_logp_difference/mean": 0.02335107885301113, "step": 113, "step_time": 1.983701549237594 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12.0, "completions/max_terminated_length": 12.0, "completions/mean_length": 10.559999465942383, "completions/mean_terminated_length": 10.559999465942383, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.1528867155313492, "epoch": 0.30978260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.020327620208263397, "learning_rate": 5e-05, "loss": -0.00013320728612598032, "num_tokens": 5213040.0, "reward": 0.23484374582767487, "reward_std": 0.4310663938522339, "rewards/correctness/mean": 0.23999999463558197, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.005156250204890966, "rewards/length_penalty/std": 0.0005594291142188013, "sampling/importance_sampling_ratio/max": 1.233178734779358, "sampling/importance_sampling_ratio/mean": 0.9935439229011536, "sampling/importance_sampling_ratio/min": 0.3149997293949127, "sampling/sampling_logp_difference/max": 1.1551835536956787, "sampling/sampling_logp_difference/mean": 0.02408069185912609, "step": 114, "step_time": 1.3785158388782293 }, { "clip_ratio/high_max": 0.0016393441706895827, "clip_ratio/high_mean": 0.0016393441706895827, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016393441706895827, "completions/clipped_ratio": 0.0, "completions/max_length": 44.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 13.75999927520752, "completions/mean_terminated_length": 13.75999927520752, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.19208596646785736, "epoch": 0.3125, "frac_reward_zero_std": 0.0, "grad_norm": 0.014285330660641193, "learning_rate": 5e-05, "loss": 0.0012205843813717365, "num_tokens": 5215958.0, "reward": 0.1732812523841858, "reward_std": 0.3881458044052124, "rewards/correctness/mean": 0.18000000715255737, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.006718750111758709, "rewards/length_penalty/std": 0.002871919423341751, "sampling/importance_sampling_ratio/max": 2.1067698001861572, "sampling/importance_sampling_ratio/mean": 0.9910176396369934, "sampling/importance_sampling_ratio/min": 0.18786033987998962, "sampling/sampling_logp_difference/max": 1.6720564365386963, "sampling/sampling_logp_difference/mean": 0.035164862871170044, "step": 115, "step_time": 1.5596709728706628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 29.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 10.179999351501465, "completions/mean_terminated_length": 10.179999351501465, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.09898749366402626, "epoch": 0.31521739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.011361517012119293, "learning_rate": 5e-05, "loss": 0.0010609884047880769, "num_tokens": 5219357.0, "reward": 0.3950292766094208, "reward_std": 0.49528878927230835, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.004970703274011612, "rewards/length_penalty/std": 0.001571226166561246, "sampling/importance_sampling_ratio/max": 1.1920545101165771, "sampling/importance_sampling_ratio/mean": 0.9905608296394348, "sampling/importance_sampling_ratio/min": 0.6660861372947693, "sampling/sampling_logp_difference/max": 0.40633630752563477, "sampling/sampling_logp_difference/mean": 0.014935722574591637, "step": 116, "step_time": 1.4674468878656626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 41.0, "completions/max_terminated_length": 41.0, "completions/mean_length": 11.279999732971191, "completions/mean_terminated_length": 11.279999732971191, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.07753250487148762, "epoch": 0.3179347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.021389001980423927, "learning_rate": 5e-05, "loss": 0.0013631616020575166, "num_tokens": 5222101.0, "reward": 0.39449217915534973, "reward_std": 0.4954357147216797, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.0055078123696148396, "rewards/length_penalty/std": 0.0029214380774646997, "sampling/importance_sampling_ratio/max": 1.5011656284332275, "sampling/importance_sampling_ratio/mean": 0.9932869672775269, "sampling/importance_sampling_ratio/min": 0.22313006222248077, "sampling/sampling_logp_difference/max": 1.5000004768371582, "sampling/sampling_logp_difference/mean": 0.02301364578306675, "step": 117, "step_time": 1.4666734847705811 }, { "clip_ratio/high_max": 0.0010204081423580646, "clip_ratio/high_mean": 0.0010204081423580646, "clip_ratio/low_mean": 0.0025129454210400582, "clip_ratio/low_min": 0.0025129454210400582, "clip_ratio/region_mean": 0.003533353563398123, "completions/clipped_ratio": 0.0, "completions/max_length": 85.0, "completions/max_terminated_length": 85.0, "completions/mean_length": 17.34000015258789, "completions/mean_terminated_length": 17.34000015258789, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.15508841276168822, "epoch": 0.32065217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.01637706533074379, "learning_rate": 5e-05, "loss": 0.0006858168635517359, "num_tokens": 5226648.0, "reward": 0.3515332043170929, "reward_std": 0.4793729782104492, "rewards/correctness/mean": 0.36000001430511475, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.008466796949505806, "rewards/length_penalty/std": 0.007427203468978405, "sampling/importance_sampling_ratio/max": 2.323932409286499, "sampling/importance_sampling_ratio/mean": 0.9894140362739563, "sampling/importance_sampling_ratio/min": 0.2500162422657013, "sampling/sampling_logp_difference/max": 1.386229395866394, "sampling/sampling_logp_difference/mean": 0.03073703683912754, "step": 118, "step_time": 1.8715046828147024 }, { "clip_ratio/high_max": 0.001431485399371013, "clip_ratio/high_mean": 0.001431485399371013, "clip_ratio/low_mean": 0.0021437447983771564, "clip_ratio/low_min": 0.0021437447983771564, "clip_ratio/region_mean": 0.0035752301977481694, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 77.0, "completions/mean_length": 65.26000213623047, "completions/mean_terminated_length": 24.795917510986328, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.17566183917224407, "epoch": 0.3233695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.013526733964681625, "learning_rate": 5e-05, "loss": 0.08021794259548187, "num_tokens": 5233421.0, "reward": 0.3281347453594208, "reward_std": 0.5138391852378845, "rewards/correctness/mean": 0.36000001430511475, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.031865235418081284, "rewards/length_penalty/std": 0.14017029106616974, "sampling/importance_sampling_ratio/max": 1.5155799388885498, "sampling/importance_sampling_ratio/mean": 0.9970149397850037, "sampling/importance_sampling_ratio/min": 0.48622196912765503, "sampling/sampling_logp_difference/max": 0.7210900187492371, "sampling/sampling_logp_difference/mean": 0.009188398718833923, "step": 119, "step_time": 20.337825877359137 }, { "clip_ratio/high_max": 0.002040816284716129, "clip_ratio/high_mean": 0.002040816284716129, "clip_ratio/low_mean": 0.0021739130839705466, "clip_ratio/low_min": 0.0021739130839705466, "clip_ratio/region_mean": 0.004214729368686676, "completions/clipped_ratio": 0.0, "completions/max_length": 18.0, "completions/max_terminated_length": 18.0, "completions/mean_length": 10.380000114440918, "completions/mean_terminated_length": 10.380000114440918, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.25275845229625704, "epoch": 0.32608695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.010559819638729095, "learning_rate": 5e-05, "loss": 0.00029482506215572357, "num_tokens": 5236790.0, "reward": 0.07493163645267487, "reward_std": 0.27439793944358826, "rewards/correctness/mean": 0.07999999821186066, "rewards/correctness/std": 0.27404749393463135, "rewards/length_penalty/mean": -0.0050683594308793545, "rewards/length_penalty/std": 0.0015687467530369759, "sampling/importance_sampling_ratio/max": 1.5421497821807861, "sampling/importance_sampling_ratio/mean": 0.9877521991729736, "sampling/importance_sampling_ratio/min": 0.38020059466362, "sampling/sampling_logp_difference/max": 0.9670562744140625, "sampling/sampling_logp_difference/mean": 0.03305759280920029, "step": 120, "step_time": 1.3726711603812873 }, { "clip_ratio/high_max": 0.004678522609174252, "clip_ratio/high_mean": 0.004678522609174252, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004678522609174252, "completions/clipped_ratio": 0.0, "completions/max_length": 22.0, "completions/max_terminated_length": 22.0, "completions/mean_length": 9.319999694824219, "completions/mean_terminated_length": 9.319999694824219, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.06333770230412483, "epoch": 0.328804347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.036573126912117004, "learning_rate": 5e-05, "loss": 0.0010053031146526337, "num_tokens": 5240596.0, "reward": 0.3954492211341858, "reward_std": 0.49500203132629395, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.004550781100988388, "rewards/length_penalty/std": 0.0012298959773033857, "sampling/importance_sampling_ratio/max": 1.5860633850097656, "sampling/importance_sampling_ratio/mean": 0.9944274425506592, "sampling/importance_sampling_ratio/min": 0.5976189970970154, "sampling/sampling_logp_difference/max": 0.5148018598556519, "sampling/sampling_logp_difference/mean": 0.011470520868897438, "step": 121, "step_time": 1.4338487619534135 }, { "clip_ratio/high_max": 0.0021276595070958138, "clip_ratio/high_mean": 0.0021276595070958138, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021276595070958138, "completions/clipped_ratio": 0.0, "completions/max_length": 12.0, "completions/max_terminated_length": 12.0, "completions/mean_length": 10.09999942779541, "completions/mean_terminated_length": 10.09999942779541, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.1215081237256527, "epoch": 0.33152173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.024983931332826614, "learning_rate": 5e-05, "loss": 0.0001665241434238851, "num_tokens": 5244221.0, "reward": 0.1350683569908142, "reward_std": 0.350132554769516, "rewards/correctness/mean": 0.14000000059604645, "rewards/correctness/std": 0.3505098223686218, "rewards/length_penalty/mean": -0.004931640811264515, "rewards/length_penalty/std": 0.0008614040561951697, "sampling/importance_sampling_ratio/max": 2.7834041118621826, "sampling/importance_sampling_ratio/mean": 0.9930852651596069, "sampling/importance_sampling_ratio/min": 0.43255409598350525, "sampling/sampling_logp_difference/max": 1.023674726486206, "sampling/sampling_logp_difference/mean": 0.02821943163871765, "step": 122, "step_time": 1.373616496566683 }, { "clip_ratio/high_max": 0.0035325242206454277, "clip_ratio/high_mean": 0.0035325242206454277, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035325242206454277, "completions/clipped_ratio": 0.0, "completions/max_length": 17.0, "completions/max_terminated_length": 17.0, "completions/mean_length": 11.5, "completions/mean_terminated_length": 11.5, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.12081824392080306, "epoch": 0.3342391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.019469941034913063, "learning_rate": 5e-05, "loss": 0.0006497701397165656, "num_tokens": 5248036.0, "reward": -0.005615234375, "reward_std": 0.001525853993371129, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.005615234375, "rewards/length_penalty/std": 0.001525853993371129, "sampling/importance_sampling_ratio/max": 1.4670969247817993, "sampling/importance_sampling_ratio/mean": 0.9967119693756104, "sampling/importance_sampling_ratio/min": 0.07796137034893036, "sampling/sampling_logp_difference/max": 2.551541805267334, "sampling/sampling_logp_difference/mean": 0.03180255368351936, "step": 123, "step_time": 1.416173886274919 }, { "clip_ratio/high_max": 0.009455096907913685, "clip_ratio/high_mean": 0.009455096907913685, "clip_ratio/low_mean": 0.0034365782514214514, "clip_ratio/low_min": 0.0034365782514214514, "clip_ratio/region_mean": 0.012891675159335137, "completions/clipped_ratio": 0.0, "completions/max_length": 20.0, "completions/max_terminated_length": 20.0, "completions/mean_length": 10.859999656677246, "completions/mean_terminated_length": 10.859999656677246, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.11404812037944793, "epoch": 0.33695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.024839911609888077, "learning_rate": 5e-05, "loss": -0.0001064792595570907, "num_tokens": 5250829.0, "reward": 0.45469725131988525, "reward_std": 0.5023077726364136, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.005302734207361937, "rewards/length_penalty/std": 0.0019974883180111647, "sampling/importance_sampling_ratio/max": 1.6004278659820557, "sampling/importance_sampling_ratio/mean": 0.9722920656204224, "sampling/importance_sampling_ratio/min": 0.01483890786767006, "sampling/sampling_logp_difference/max": 4.210502624511719, "sampling/sampling_logp_difference/mean": 0.08302152901887894, "step": 124, "step_time": 1.3816925161518157 }, { "clip_ratio/high_max": 0.0033976585022173823, "clip_ratio/high_mean": 0.0033976585022173823, "clip_ratio/low_mean": 0.002684563770890236, "clip_ratio/low_min": 0.002684563770890236, "clip_ratio/region_mean": 0.006082222273107618, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 63.0, "completions/mean_length": 52.39999771118164, "completions/mean_terminated_length": 11.673469543457031, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.06118136048316956, "epoch": 0.33967391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.014367754571139812, "learning_rate": 5e-05, "loss": 0.07726050913333893, "num_tokens": 5255929.0, "reward": 0.3744140565395355, "reward_std": 0.5283631682395935, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.02558593824505806, "rewards/length_penalty/std": 0.1407146453857422, "sampling/importance_sampling_ratio/max": 2.106475353240967, "sampling/importance_sampling_ratio/mean": 0.9985984563827515, "sampling/importance_sampling_ratio/min": 0.3653067648410797, "sampling/sampling_logp_difference/max": 1.0070178508758545, "sampling/sampling_logp_difference/mean": 0.004823117051273584, "step": 125, "step_time": 20.179652540944517 }, { "clip_ratio/high_max": 0.00827543493360281, "clip_ratio/high_mean": 0.00827543493360281, "clip_ratio/low_mean": 0.002083333395421505, "clip_ratio/low_min": 0.002083333395421505, "clip_ratio/region_mean": 0.010358768329024316, "completions/clipped_ratio": 0.0, "completions/max_length": 14.0, "completions/max_terminated_length": 14.0, "completions/mean_length": 10.199999809265137, "completions/mean_terminated_length": 10.199999809265137, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.1312358409166336, "epoch": 0.3423913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.010240713134407997, "learning_rate": 5e-05, "loss": 9.463776223128662e-05, "num_tokens": 5261409.0, "reward": 0.05501953139901161, "reward_std": 0.23984263837337494, "rewards/correctness/mean": 0.05999999865889549, "rewards/correctness/std": 0.2398979365825653, "rewards/length_penalty/mean": -0.0049804686568677425, "rewards/length_penalty/std": 0.0011713765561580658, "sampling/importance_sampling_ratio/max": 2.973083257675171, "sampling/importance_sampling_ratio/mean": 0.9844138622283936, "sampling/importance_sampling_ratio/min": 0.2322712540626526, "sampling/sampling_logp_difference/max": 1.4598493576049805, "sampling/sampling_logp_difference/mean": 0.05247528478503227, "step": 126, "step_time": 1.426745121134445 }, { "clip_ratio/high_max": 0.0015625, "clip_ratio/high_mean": 0.0015625, "clip_ratio/low_mean": 0.010037288442254067, "clip_ratio/low_min": 0.010037288442254067, "clip_ratio/region_mean": 0.011599788442254066, "completions/clipped_ratio": 0.0, "completions/max_length": 18.0, "completions/max_terminated_length": 18.0, "completions/mean_length": 11.519999504089355, "completions/mean_terminated_length": 11.519999504089355, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.12240712642669678, "epoch": 0.3451086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.01998254843056202, "learning_rate": 5e-05, "loss": 0.0006313637131825089, "num_tokens": 5264615.0, "reward": -0.005624999757856131, "reward_std": 0.001768079586327076, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.005625000223517418, "rewards/length_penalty/std": 0.001768079586327076, "sampling/importance_sampling_ratio/max": 1.4942423105239868, "sampling/importance_sampling_ratio/mean": 0.9801560640335083, "sampling/importance_sampling_ratio/min": 0.02733716182410717, "sampling/sampling_logp_difference/max": 3.599508285522461, "sampling/sampling_logp_difference/mean": 0.06355856359004974, "step": 127, "step_time": 1.4069360962603241 }, { "clip_ratio/high_max": 0.005801975913345814, "clip_ratio/high_mean": 0.005801975913345814, "clip_ratio/low_mean": 0.008586328104138374, "clip_ratio/low_min": 0.008586328104138374, "clip_ratio/region_mean": 0.014388304017484189, "completions/clipped_ratio": 0.0, "completions/max_length": 10.0, "completions/max_terminated_length": 10.0, "completions/mean_length": 7.21999979019165, "completions/mean_terminated_length": 7.21999979019165, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.14161121100187302, "epoch": 0.34782608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.016605541110038757, "learning_rate": 5e-05, "loss": 0.0001308184873778373, "num_tokens": 5267296.0, "reward": 0.056474607437849045, "reward_std": 0.23955321311950684, "rewards/correctness/mean": 0.05999999865889549, "rewards/correctness/std": 0.2398979514837265, "rewards/length_penalty/mean": -0.003525390522554517, "rewards/length_penalty/std": 0.0009680049843154848, "sampling/importance_sampling_ratio/max": 2.0141804218292236, "sampling/importance_sampling_ratio/mean": 0.9972620010375977, "sampling/importance_sampling_ratio/min": 0.2654344439506531, "sampling/sampling_logp_difference/max": 1.3263874053955078, "sampling/sampling_logp_difference/mean": 0.04654447361826897, "step": 128, "step_time": 1.34115265798755 }, { "clip_ratio/high_max": 0.0052564961835741995, "clip_ratio/high_mean": 0.0052564961835741995, "clip_ratio/low_mean": 0.0030814639292657377, "clip_ratio/low_min": 0.0030814639292657377, "clip_ratio/region_mean": 0.008337960112839937, "completions/clipped_ratio": 0.0, "completions/max_length": 29.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 11.960000038146973, "completions/mean_terminated_length": 11.960000038146973, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.145098540186882, "epoch": 0.35054347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.019570035859942436, "learning_rate": 5e-05, "loss": 0.001352812978439033, "num_tokens": 5270954.0, "reward": -0.0058398437686264515, "reward_std": 0.0026634151581674814, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0058398437686264515, "rewards/length_penalty/std": 0.0026634151581674814, "sampling/importance_sampling_ratio/max": 2.7703750133514404, "sampling/importance_sampling_ratio/mean": 0.9858039021492004, "sampling/importance_sampling_ratio/min": 0.4572809338569641, "sampling/sampling_logp_difference/max": 1.0189826488494873, "sampling/sampling_logp_difference/mean": 0.03138452768325806, "step": 129, "step_time": 1.4406253728084266 }, { "clip_ratio/high_max": 0.023539825342595577, "clip_ratio/high_mean": 0.023539825342595577, "clip_ratio/low_mean": 0.01050705425441265, "clip_ratio/low_min": 0.01050705425441265, "clip_ratio/region_mean": 0.03404687978327274, "completions/clipped_ratio": 0.0, "completions/max_length": 10.0, "completions/max_terminated_length": 10.0, "completions/mean_length": 7.159999847412109, "completions/mean_terminated_length": 7.159999847412109, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.28884335458278654, "epoch": 0.3532608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.024998050183057785, "learning_rate": 5e-05, "loss": 0.0004798858717549592, "num_tokens": 5274082.0, "reward": 0.3765038847923279, "reward_std": 0.4897673428058624, "rewards/correctness/mean": 0.3799999952316284, "rewards/correctness/std": 0.4903143346309662, "rewards/length_penalty/mean": -0.003496093675494194, "rewards/length_penalty/std": 0.0010867377277463675, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9742715954780579, "sampling/importance_sampling_ratio/min": 0.09587825834751129, "sampling/sampling_logp_difference/max": 2.3446760177612305, "sampling/sampling_logp_difference/mean": 0.13332518935203552, "step": 130, "step_time": 1.328675802797079 }, { "clip_ratio/high_max": 0.003103244863450527, "clip_ratio/high_mean": 0.003103244863450527, "clip_ratio/low_mean": 0.0033828146755695344, "clip_ratio/low_min": 0.0033828146755695344, "clip_ratio/region_mean": 0.006486059539020062, "completions/clipped_ratio": 0.0, "completions/max_length": 62.0, "completions/max_terminated_length": 62.0, "completions/mean_length": 9.539999961853027, "completions/mean_terminated_length": 9.539999961853027, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.23320847153663635, "epoch": 0.35597826086956524, "frac_reward_zero_std": 0.0, "grad_norm": 0.017455505207180977, "learning_rate": 5e-05, "loss": 0.003064260818064213, "num_tokens": 5277939.0, "reward": -0.0046582031063735485, "reward_std": 0.006479442585259676, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0046582031063735485, "rewards/length_penalty/std": 0.006479442585259676, "sampling/importance_sampling_ratio/max": 1.6986380815505981, "sampling/importance_sampling_ratio/mean": 0.9788010716438293, "sampling/importance_sampling_ratio/min": 0.15788955986499786, "sampling/sampling_logp_difference/max": 1.8458595275878906, "sampling/sampling_logp_difference/mean": 0.05766844004392624, "step": 131, "step_time": 1.6266542652156204 }, { "clip_ratio/high_max": 9.46073792874813e-05, "clip_ratio/high_mean": 9.46073792874813e-05, "clip_ratio/low_mean": 0.00634920671582222, "clip_ratio/low_min": 0.00634920671582222, "clip_ratio/region_mean": 0.006443814095109701, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 13.0, "completions/mean_length": 47.20000076293945, "completions/mean_terminated_length": 6.36734676361084, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.20117006599903106, "epoch": 0.358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.021256737411022186, "learning_rate": 5e-05, "loss": 0.060951780527830124, "num_tokens": 5282129.0, "reward": -0.003046874888241291, "reward_std": 0.20168958604335785, "rewards/correctness/mean": 0.019999999552965164, "rewards/correctness/std": 0.1414213627576828, "rewards/length_penalty/mean": -0.02304687537252903, "rewards/length_penalty/std": 0.14098763465881348, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9934961199760437, "sampling/importance_sampling_ratio/min": 3.782039357247413e-07, "sampling/sampling_logp_difference/max": 14.787832260131836, "sampling/sampling_logp_difference/mean": 0.04299402981996536, "step": 132, "step_time": 19.68206503125839 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 4.199999809265137, "completions/mean_terminated_length": 4.199999809265137, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 0.23295619785785676, "epoch": 0.36141304347826086, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5285029.0, "reward": 0.19794921576976776, "reward_std": 0.40435701608657837, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.0020507811568677425, "rewards/length_penalty/std": 0.00036910592461936176, "sampling/importance_sampling_ratio/max": 1.188761591911316, "sampling/importance_sampling_ratio/mean": 0.9596621990203857, "sampling/importance_sampling_ratio/min": 0.024618901312351227, "sampling/sampling_logp_difference/max": 3.7042407989501953, "sampling/sampling_logp_difference/mean": 0.12237008661031723, "step": 133, "step_time": 1.264468238921836 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0054054055362939835, "clip_ratio/low_min": 0.0054054055362939835, "clip_ratio/region_mean": 0.0054054055362939835, "completions/clipped_ratio": 0.0, "completions/max_length": 6.0, "completions/max_terminated_length": 6.0, "completions/mean_length": 3.319999933242798, "completions/mean_terminated_length": 3.319999933242798, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17164091616868973, "epoch": 0.3641304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.006110444199293852, "learning_rate": 5e-05, "loss": 0.00015725757111795247, "num_tokens": 5287885.0, "reward": 0.19837890565395355, "reward_std": 0.4043866693973541, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.0016210937174037099, "rewards/length_penalty/std": 0.00043494580313563347, "sampling/importance_sampling_ratio/max": 1.356533169746399, "sampling/importance_sampling_ratio/mean": 0.9950489401817322, "sampling/importance_sampling_ratio/min": 0.0008358496124856174, "sampling/sampling_logp_difference/max": 7.087061882019043, "sampling/sampling_logp_difference/mean": 0.07953914254903793, "step": 134, "step_time": 1.3009774491656572 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0035087719559669496, "clip_ratio/low_min": 0.0035087719559669496, "clip_ratio/region_mean": 0.0035087719559669496, "completions/clipped_ratio": 0.0, "completions/max_length": 12.0, "completions/max_terminated_length": 12.0, "completions/mean_length": 4.599999904632568, "completions/mean_terminated_length": 4.599999904632568, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2973464459180832, "epoch": 0.36684782608695654, "frac_reward_zero_std": 0.0, "grad_norm": 0.011810951866209507, "learning_rate": 5e-05, "loss": 0.0003894786350429058, "num_tokens": 5292295.0, "reward": -0.0022460937034338713, "reward_std": 0.0009665461839176714, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0022460937034338713, "rewards/length_penalty/std": 0.0009665461839176714, "sampling/importance_sampling_ratio/max": 1.6216562986373901, "sampling/importance_sampling_ratio/mean": 0.9891418218612671, "sampling/importance_sampling_ratio/min": 0.11668897420167923, "sampling/sampling_logp_difference/max": 2.1482431888580322, "sampling/sampling_logp_difference/mean": 0.07455261051654816, "step": 135, "step_time": 1.3720859428867698 }, { "clip_ratio/high_max": 0.01212609987705946, "clip_ratio/high_mean": 0.01212609987705946, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.01212609987705946, "completions/clipped_ratio": 0.0, "completions/max_length": 107.0, "completions/max_terminated_length": 107.0, "completions/mean_length": 6.420000076293945, "completions/mean_terminated_length": 6.420000076293945, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.4212904691696167, "epoch": 0.3695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.009215766564011574, "learning_rate": 5e-05, "loss": 0.0023378136102110147, "num_tokens": 5294696.0, "reward": 0.21686522662639618, "reward_std": 0.419146329164505, "rewards/correctness/mean": 0.2199999988079071, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.003134765662252903, "rewards/length_penalty/std": 0.0071464055217802525, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9935191869735718, "sampling/importance_sampling_ratio/min": 0.30233317613601685, "sampling/sampling_logp_difference/max": 1.2943968772888184, "sampling/sampling_logp_difference/mean": 0.08518271148204803, "step": 136, "step_time": 1.852066189981997 }, { "clip_ratio/high_max": 0.00416666679084301, "clip_ratio/high_mean": 0.00416666679084301, "clip_ratio/low_mean": 0.016872549057006837, "clip_ratio/low_min": 0.016872549057006837, "clip_ratio/region_mean": 0.021039215847849846, "completions/clipped_ratio": 0.0, "completions/max_length": 26.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 6.71999979019165, "completions/mean_terminated_length": 6.71999979019165, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.2728432804346085, "epoch": 0.37228260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.027852702885866165, "learning_rate": 5e-05, "loss": 0.0009244568645954132, "num_tokens": 5297892.0, "reward": 0.3567187488079071, "reward_std": 0.4856407046318054, "rewards/correctness/mean": 0.36000001430511475, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.003281249897554517, "rewards/length_penalty/std": 0.0028727324679493904, "sampling/importance_sampling_ratio/max": 1.9078514575958252, "sampling/importance_sampling_ratio/mean": 0.9902607202529907, "sampling/importance_sampling_ratio/min": 0.32021790742874146, "sampling/sampling_logp_difference/max": 1.1387535333633423, "sampling/sampling_logp_difference/mean": 0.07158994674682617, "step": 137, "step_time": 1.4134835042059422 }, { "clip_ratio/high_max": 0.016225016117095946, "clip_ratio/high_mean": 0.016225016117095946, "clip_ratio/low_mean": 0.00416666679084301, "clip_ratio/low_min": 0.00416666679084301, "clip_ratio/region_mean": 0.020391682535409926, "completions/clipped_ratio": 0.0, "completions/max_length": 15.0, "completions/max_terminated_length": 15.0, "completions/mean_length": 5.079999923706055, "completions/mean_terminated_length": 5.079999923706055, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.17912932634353637, "epoch": 0.375, "frac_reward_zero_std": 0.0, "grad_norm": 0.017889760434627533, "learning_rate": 5e-05, "loss": 0.00015456334222108126, "num_tokens": 5301176.0, "reward": 0.15751953423023224, "reward_std": 0.3685983121395111, "rewards/correctness/mean": 0.1599999964237213, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.002480468712747097, "rewards/length_penalty/std": 0.0021268592681735754, "sampling/importance_sampling_ratio/max": 1.55757737159729, "sampling/importance_sampling_ratio/mean": 0.9653849005699158, "sampling/importance_sampling_ratio/min": 0.3918115496635437, "sampling/sampling_logp_difference/max": 0.936974287033081, "sampling/sampling_logp_difference/mean": 0.05680634453892708, "step": 138, "step_time": 1.783718638587743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4.0, "completions/max_terminated_length": 4.0, "completions/mean_length": 2.5, "completions/mean_terminated_length": 2.5, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.11814753264188767, "epoch": 0.37771739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.01109712291508913, "learning_rate": 5e-05, "loss": 3.969172394135967e-05, "num_tokens": 5304081.0, "reward": 0.2187792956829071, "reward_std": 0.4180828928947449, "rewards/correctness/mean": 0.2199999988079071, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.001220703125, "rewards/length_penalty/std": 0.0004214232030790299, "sampling/importance_sampling_ratio/max": 1.2242869138717651, "sampling/importance_sampling_ratio/mean": 0.9961508512496948, "sampling/importance_sampling_ratio/min": 0.16473731398582458, "sampling/sampling_logp_difference/max": 1.8034031391143799, "sampling/sampling_logp_difference/mean": 0.055593159049749374, "step": 139, "step_time": 1.2756988110486418 }, { "clip_ratio/high_max": 0.00625, "clip_ratio/high_mean": 0.00625, "clip_ratio/low_mean": 0.01653299927711487, "clip_ratio/low_min": 0.01653299927711487, "clip_ratio/region_mean": 0.022782999277114867, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 3.5399999618530273, "completions/mean_terminated_length": 3.5399999618530273, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.21788379549980164, "epoch": 0.3804347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.014283841475844383, "learning_rate": 5e-05, "loss": 7.27055812603794e-05, "num_tokens": 5307898.0, "reward": 0.01827148348093033, "reward_std": 0.14146029949188232, "rewards/correctness/mean": 0.019999999552965164, "rewards/correctness/std": 0.1414213627576828, "rewards/length_penalty/mean": -0.0017285156063735485, "rewards/length_penalty/std": 0.0005050338804721832, "sampling/importance_sampling_ratio/max": 2.1249887943267822, "sampling/importance_sampling_ratio/mean": 0.980110228061676, "sampling/importance_sampling_ratio/min": 0.34859681129455566, "sampling/sampling_logp_difference/max": 1.0538392066955566, "sampling/sampling_logp_difference/mean": 0.04972858726978302, "step": 140, "step_time": 1.3094815532676876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4.0, "completions/max_terminated_length": 4.0, "completions/mean_length": 3.419999837875366, "completions/mean_terminated_length": 3.419999837875366, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 0.030646063201129437, "epoch": 0.38315217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.004051357973366976, "learning_rate": 5e-05, "loss": 2.7182053599972278e-05, "num_tokens": 5310799.0, "reward": 0.39833006262779236, "reward_std": 0.4946380853652954, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.001669921912252903, "rewards/length_penalty/std": 0.0002434420894132927, "sampling/importance_sampling_ratio/max": 1.028769850730896, "sampling/importance_sampling_ratio/mean": 0.9956688284873962, "sampling/importance_sampling_ratio/min": 0.4431457221508026, "sampling/sampling_logp_difference/max": 0.8138566017150879, "sampling/sampling_logp_difference/mean": 0.008249584585428238, "step": 141, "step_time": 1.2636189230252057 }, { "clip_ratio/high_max": 0.006451612710952759, "clip_ratio/high_mean": 0.006451612710952759, "clip_ratio/low_mean": 0.013118279725313186, "clip_ratio/low_min": 0.013118279725313186, "clip_ratio/region_mean": 0.019569892436265945, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 3.179999828338623, "completions/mean_terminated_length": 3.179999828338623, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.41223096251487734, "epoch": 0.3858695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.007143351249396801, "learning_rate": 5e-05, "loss": -4.87293436890468e-05, "num_tokens": 5313748.0, "reward": -0.0015527342911809683, "reward_std": 0.0005640199524350464, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0015527344075962901, "rewards/length_penalty/std": 0.0005640199524350464, "sampling/importance_sampling_ratio/max": 2.4648897647857666, "sampling/importance_sampling_ratio/mean": 0.9650440812110901, "sampling/importance_sampling_ratio/min": 0.28366824984550476, "sampling/sampling_logp_difference/max": 1.259949803352356, "sampling/sampling_logp_difference/mean": 0.11519324779510498, "step": 142, "step_time": 1.2917137211188674 }, { "clip_ratio/high_max": 0.003999999910593033, "clip_ratio/high_mean": 0.003999999910593033, "clip_ratio/low_mean": 0.003999999910593033, "clip_ratio/low_min": 0.003999999910593033, "clip_ratio/region_mean": 0.007999999821186066, "completions/clipped_ratio": 0.0, "completions/max_length": 12.0, "completions/max_terminated_length": 12.0, "completions/mean_length": 4.119999885559082, "completions/mean_terminated_length": 4.119999885559082, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.14669495075941086, "epoch": 0.38858695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.03668024763464928, "learning_rate": 5e-05, "loss": -1.1713493222487159e-05, "num_tokens": 5316294.0, "reward": 0.4979882836341858, "reward_std": 0.50434809923172, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.0020117186941206455, "rewards/length_penalty/std": 0.0015334563795477152, "sampling/importance_sampling_ratio/max": 1.1682599782943726, "sampling/importance_sampling_ratio/mean": 0.9931513071060181, "sampling/importance_sampling_ratio/min": 0.3147142827510834, "sampling/sampling_logp_difference/max": 1.1560900211334229, "sampling/sampling_logp_difference/mean": 0.03275330364704132, "step": 143, "step_time": 1.3322547457646579 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12.0, "completions/max_terminated_length": 12.0, "completions/mean_length": 4.400000095367432, "completions/mean_terminated_length": 4.400000095367432, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.28117292523384096, "epoch": 0.391304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.014653222635388374, "learning_rate": 5e-05, "loss": 0.0007023264188319445, "num_tokens": 5319564.0, "reward": 0.19785155355930328, "reward_std": 0.4046550989151001, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.0021484375465661287, "rewards/length_penalty/std": 0.0014631820376962423, "sampling/importance_sampling_ratio/max": 1.436386227607727, "sampling/importance_sampling_ratio/mean": 0.982090175151825, "sampling/importance_sampling_ratio/min": 0.49318453669548035, "sampling/sampling_logp_difference/max": 0.7068718671798706, "sampling/sampling_logp_difference/mean": 0.03635406866669655, "step": 144, "step_time": 1.3282462656497955 }, { "clip_ratio/high_max": 0.00555555559694767, "clip_ratio/high_mean": 0.00555555559694767, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00555555559694767, "completions/clipped_ratio": 0.0, "completions/max_length": 7.0, "completions/max_terminated_length": 7.0, "completions/mean_length": 4.019999980926514, "completions/mean_terminated_length": 4.019999980926514, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.06284114345908165, "epoch": 0.39402173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.009140221402049065, "learning_rate": 5e-05, "loss": 7.00859964126721e-05, "num_tokens": 5322095.0, "reward": 0.238037109375, "reward_std": 0.43184083700180054, "rewards/correctness/mean": 0.23999999463558197, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.0019628906156867743, "rewards/length_penalty/std": 0.0007147025316953659, "sampling/importance_sampling_ratio/max": 1.8910298347473145, "sampling/importance_sampling_ratio/mean": 0.9930315017700195, "sampling/importance_sampling_ratio/min": 0.3770080804824829, "sampling/sampling_logp_difference/max": 0.9754886627197266, "sampling/sampling_logp_difference/mean": 0.022765684872865677, "step": 145, "step_time": 1.3034854554571211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.008695652335882187, "clip_ratio/low_min": 0.008695652335882187, "clip_ratio/region_mean": 0.008695652335882187, "completions/clipped_ratio": 0.0, "completions/max_length": 3.0, "completions/max_terminated_length": 3.0, "completions/mean_length": 2.240000009536743, "completions/mean_terminated_length": 2.240000009536743, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.18227319419384003, "epoch": 0.3967391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.02299347147345543, "learning_rate": 5e-05, "loss": 4.629989416571334e-05, "num_tokens": 5325147.0, "reward": 0.39890623092651367, "reward_std": 0.4947669804096222, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487170577049255, "rewards/length_penalty/mean": -0.0010937500046566129, "rewards/length_penalty/std": 0.0002106538595398888, "sampling/importance_sampling_ratio/max": 1.1442818641662598, "sampling/importance_sampling_ratio/mean": 1.0006992816925049, "sampling/importance_sampling_ratio/min": 0.5129154324531555, "sampling/sampling_logp_difference/max": 0.6676442623138428, "sampling/sampling_logp_difference/mean": 0.0328700952231884, "step": 146, "step_time": 1.2859444648493081 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8.0, "completions/max_terminated_length": 8.0, "completions/mean_length": 3.1999998092651367, "completions/mean_terminated_length": 3.1999998092651367, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.043758167326450347, "epoch": 0.39945652173913043, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5327607.0, "reward": 0.3984375, "reward_std": 0.4941476285457611, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.0015625000232830644, "rewards/length_penalty/std": 0.0011837725760415196, "sampling/importance_sampling_ratio/max": 1.0770561695098877, "sampling/importance_sampling_ratio/mean": 0.9971351623535156, "sampling/importance_sampling_ratio/min": 0.36504772305488586, "sampling/sampling_logp_difference/max": 1.0077271461486816, "sampling/sampling_logp_difference/mean": 0.016550671309232712, "step": 147, "step_time": 1.3252897900529206 }, { "clip_ratio/high_max": 0.007999999821186066, "clip_ratio/high_mean": 0.007999999821186066, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007999999821186066, "completions/clipped_ratio": 0.0, "completions/max_length": 3.0, "completions/max_terminated_length": 3.0, "completions/mean_length": 2.5999999046325684, "completions/mean_terminated_length": 2.5999999046325684, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.24429004788398742, "epoch": 0.40217391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.025154076516628265, "learning_rate": 5e-05, "loss": 7.58409732952714e-05, "num_tokens": 5330647.0, "reward": 0.15873046219348907, "reward_std": 0.37024199962615967, "rewards/correctness/mean": 0.1599999964237213, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.0012695312034338713, "rewards/length_penalty/std": 0.00024163654597941786, "sampling/importance_sampling_ratio/max": 1.84139084815979, "sampling/importance_sampling_ratio/mean": 1.0035784244537354, "sampling/importance_sampling_ratio/min": 0.2585758864879608, "sampling/sampling_logp_difference/max": 1.352566123008728, "sampling/sampling_logp_difference/mean": 0.08900177478790283, "step": 148, "step_time": 1.2713856103364378 }, { "clip_ratio/high_max": 0.004894911497831345, "clip_ratio/high_mean": 0.004894911497831345, "clip_ratio/low_mean": 0.00786290317773819, "clip_ratio/low_min": 0.00786290317773819, "clip_ratio/region_mean": 0.012757814675569534, "completions/clipped_ratio": 0.0, "completions/max_length": 57.0, "completions/max_terminated_length": 57.0, "completions/mean_length": 9.779999732971191, "completions/mean_terminated_length": 9.779999732971191, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3053502798080444, "epoch": 0.4048913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.021597005426883698, "learning_rate": 5e-05, "loss": 0.0015805705916136503, "num_tokens": 5334106.0, "reward": 0.21522460877895355, "reward_std": 0.4204765558242798, "rewards/correctness/mean": 0.2199999988079071, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.0047753904946148396, "rewards/length_penalty/std": 0.007258681580424309, "sampling/importance_sampling_ratio/max": 1.8426822423934937, "sampling/importance_sampling_ratio/mean": 0.9776626825332642, "sampling/importance_sampling_ratio/min": 0.38441887497901917, "sampling/sampling_logp_difference/max": 0.9560225009918213, "sampling/sampling_logp_difference/mean": 0.0729394257068634, "step": 149, "step_time": 1.6369031788781285 }, { "clip_ratio/high_max": 0.006666667014360428, "clip_ratio/high_mean": 0.006666667014360428, "clip_ratio/low_mean": 0.05519783571362495, "clip_ratio/low_min": 0.05519783571362495, "clip_ratio/region_mean": 0.061864501982927325, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 2.879999876022339, "completions/mean_terminated_length": 2.879999876022339, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.16436495780944824, "epoch": 0.4076086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.021375233307480812, "learning_rate": 5e-05, "loss": 0.00015874237578827888, "num_tokens": 5336010.0, "reward": 0.5785937309265137, "reward_std": 0.49848005175590515, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.0014062500558793545, "rewards/length_penalty/std": 0.00046938066952861845, "sampling/importance_sampling_ratio/max": 1.5337551832199097, "sampling/importance_sampling_ratio/mean": 0.9424799084663391, "sampling/importance_sampling_ratio/min": 0.12197969108819962, "sampling/sampling_logp_difference/max": 2.103900671005249, "sampling/sampling_logp_difference/mean": 0.1282735913991928, "step": 150, "step_time": 1.6516870979685336 } ], "logging_steps": 1, "max_steps": 200, "num_input_tokens_seen": 5336010, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 10, "trial_name": null, "trial_params": null }