{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.2717391304347826, "eval_steps": 500, "global_step": 100, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1671.2799072265625, "completions/mean_terminated_length": 1398.4827880859375, "completions/min_length": 992.0, "completions/min_terminated_length": 992.0, "entropy": 0.21248915791511536, "epoch": 0.002717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.01607886701822281, "learning_rate": 0.0, "loss": 0.07359933853149414, "num_tokens": 86304.0, "reward": -0.21605467796325684, "reward_std": 0.656523585319519, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.8160547018051147, "rewards/length_penalty/std": 0.18964654207229614, "sampling/importance_sampling_ratio/max": 1.5222556591033936, "sampling/importance_sampling_ratio/mean": 0.9927070140838623, "sampling/importance_sampling_ratio/min": 0.6435883045196533, "sampling/sampling_logp_difference/max": 0.44069600105285645, "sampling/sampling_logp_difference/mean": 0.013988969847559929, "step": 1, "step_time": 27.394495805958286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6800000071525574, "completions/max_length": 2048.0, "completions/max_terminated_length": 1991.0, "completions/mean_length": 1941.47998046875, "completions/mean_terminated_length": 1715.125, "completions/min_length": 1079.0, "completions/min_terminated_length": 1079.0, "entropy": 0.2905632793903351, "epoch": 0.005434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.01730405166745186, "learning_rate": 5e-06, "loss": 0.07159668952226639, "num_tokens": 186618.0, "reward": -0.7279882431030273, "reward_std": 0.4917677640914917, "rewards/correctness/mean": 0.2199999988079071, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.9479882717132568, "rewards/length_penalty/std": 0.10518474876880646, "sampling/importance_sampling_ratio/max": 1.450933814048767, "sampling/importance_sampling_ratio/mean": 0.9901063442230225, "sampling/importance_sampling_ratio/min": 0.3688630163669586, "sampling/sampling_logp_difference/max": 0.9973299503326416, "sampling/sampling_logp_difference/mean": 0.01780563034117222, "step": 2, "step_time": 28.991049223113805 }, { "clip_ratio/high_max": 0.0009276511613279581, "clip_ratio/high_mean": 0.0009276511613279581, "clip_ratio/low_mean": 0.00012337134103290737, "clip_ratio/low_min": 0.00012337134103290737, "clip_ratio/region_mean": 0.0010510225081816315, "completions/clipped_ratio": 0.3999999761581421, "completions/max_length": 2048.0, "completions/max_terminated_length": 2048.0, "completions/mean_length": 1714.760009765625, "completions/mean_terminated_length": 1492.60009765625, "completions/min_length": 753.0, "completions/min_terminated_length": 753.0, "entropy": 0.2737257957458496, "epoch": 0.008152173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.017563248053193092, "learning_rate": 1e-05, "loss": 0.09476927667856216, "num_tokens": 274626.0, "reward": -0.21728515625, "reward_std": 0.6381246447563171, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143346309662, "rewards/length_penalty/mean": -0.8372851610183716, "rewards/length_penalty/std": 0.20151396095752716, "sampling/importance_sampling_ratio/max": 1.426476001739502, "sampling/importance_sampling_ratio/mean": 0.9904518723487854, "sampling/importance_sampling_ratio/min": 0.6474350690841675, "sampling/sampling_logp_difference/max": 0.4347367286682129, "sampling/sampling_logp_difference/mean": 0.01698637194931507, "step": 3, "step_time": 27.53572188084945 }, { "clip_ratio/high_max": 0.0008016214531380683, "clip_ratio/high_mean": 0.0008016214531380683, "clip_ratio/low_mean": 0.00010628659802023322, "clip_ratio/low_min": 0.00010628659802023322, "clip_ratio/region_mean": 0.0009079080424271524, "completions/clipped_ratio": 0.3199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 1991.0, "completions/mean_length": 1506.5399169921875, "completions/mean_terminated_length": 1251.7353515625, "completions/min_length": 638.0, "completions/min_terminated_length": 638.0, "entropy": 0.25913241803646087, "epoch": 0.010869565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.017625654116272926, "learning_rate": 1.5e-05, "loss": 0.12047722935676575, "num_tokens": 352853.0, "reward": -0.0356152318418026, "reward_std": 0.6583986878395081, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.7356152534484863, "rewards/length_penalty/std": 0.23944713175296783, "sampling/importance_sampling_ratio/max": 1.4886895418167114, "sampling/importance_sampling_ratio/mean": 0.9911117553710938, "sampling/importance_sampling_ratio/min": 0.6022539734840393, "sampling/sampling_logp_difference/max": 0.5070760250091553, "sampling/sampling_logp_difference/mean": 0.016714798286557198, "step": 4, "step_time": 27.26718003093265 }, { "clip_ratio/high_max": 0.00031515124428551646, "clip_ratio/high_mean": 0.00031515124428551646, "clip_ratio/low_mean": 0.0001696181825536769, "clip_ratio/low_min": 0.0001696181825536769, "clip_ratio/region_mean": 0.00048476942392881026, "completions/clipped_ratio": 0.7599999904632568, "completions/max_length": 2048.0, "completions/max_terminated_length": 2022.0, "completions/mean_length": 1914.3599853515625, "completions/mean_terminated_length": 1491.166748046875, "completions/min_length": 1072.0, "completions/min_terminated_length": 1072.0, "entropy": 0.2717311054468155, "epoch": 0.01358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.018083127215504646, "learning_rate": 2e-05, "loss": 0.08188383281230927, "num_tokens": 452281.0, "reward": -0.6947460770606995, "reward_std": 0.5523213148117065, "rewards/correctness/mean": 0.23999999463558197, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.9347460865974426, "rewards/length_penalty/std": 0.13313990831375122, "sampling/importance_sampling_ratio/max": 1.7996419668197632, "sampling/importance_sampling_ratio/mean": 0.9906188249588013, "sampling/importance_sampling_ratio/min": 0.6196915507316589, "sampling/sampling_logp_difference/max": 0.5875877141952515, "sampling/sampling_logp_difference/mean": 0.01734403893351555, "step": 5, "step_time": 28.98220992088318 }, { "clip_ratio/high_max": 0.0012173810973763466, "clip_ratio/high_mean": 0.0012173810973763466, "clip_ratio/low_mean": 8.390831935685128e-05, "clip_ratio/low_min": 8.390831935685128e-05, "clip_ratio/region_mean": 0.0013012894080020488, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 2042.0, "completions/mean_length": 1670.679931640625, "completions/mean_terminated_length": 1397.4482421875, "completions/min_length": 764.0, "completions/min_terminated_length": 764.0, "entropy": 0.25660490691661836, "epoch": 0.016304347826086956, "frac_reward_zero_std": 0.0, "grad_norm": 0.016991887241601944, "learning_rate": 2.5e-05, "loss": 0.07579805701971054, "num_tokens": 538805.0, "reward": -0.23576171696186066, "reward_std": 0.6744439005851746, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.4985693693161011, "rewards/length_penalty/mean": -0.8157617449760437, "rewards/length_penalty/std": 0.21996504068374634, "sampling/importance_sampling_ratio/max": 1.4670546054840088, "sampling/importance_sampling_ratio/mean": 0.9912019371986389, "sampling/importance_sampling_ratio/min": 0.6486796736717224, "sampling/sampling_logp_difference/max": 0.4328162670135498, "sampling/sampling_logp_difference/mean": 0.01650330238044262, "step": 6, "step_time": 27.390100880060345 }, { "clip_ratio/high_max": 0.0008734886534512043, "clip_ratio/high_mean": 0.0008734886534512043, "clip_ratio/low_mean": 6.886005430715159e-05, "clip_ratio/low_min": 6.886005430715159e-05, "clip_ratio/region_mean": 0.0009423487004823983, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 1985.0, "completions/mean_length": 1602.5799560546875, "completions/mean_terminated_length": 1411.6856689453125, "completions/min_length": 862.0, "completions/min_terminated_length": 862.0, "entropy": 0.24657512605190277, "epoch": 0.019021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.018574461340904236, "learning_rate": 3e-05, "loss": 0.10817548632621765, "num_tokens": 621164.0, "reward": -0.08250976353883743, "reward_std": 0.6237230896949768, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.7825097441673279, "rewards/length_penalty/std": 0.2039300501346588, "sampling/importance_sampling_ratio/max": 1.899301528930664, "sampling/importance_sampling_ratio/mean": 0.9912979006767273, "sampling/importance_sampling_ratio/min": 0.5936238765716553, "sampling/sampling_logp_difference/max": 0.6414861679077148, "sampling/sampling_logp_difference/mean": 0.015865743160247803, "step": 7, "step_time": 27.102628064341843 }, { "clip_ratio/high_max": 0.00030049889464862646, "clip_ratio/high_mean": 0.00030049889464862646, "clip_ratio/low_mean": 0.0001223457460582722, "clip_ratio/low_min": 0.0001223457460582722, "clip_ratio/region_mean": 0.00042284463997930286, "completions/clipped_ratio": 0.5399999618530273, "completions/max_length": 2048.0, "completions/max_terminated_length": 2021.0, "completions/mean_length": 1799.3199462890625, "completions/mean_terminated_length": 1507.391357421875, "completions/min_length": 766.0, "completions/min_terminated_length": 766.0, "entropy": 0.21842886209487916, "epoch": 0.021739130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.01700979843735695, "learning_rate": 3.5e-05, "loss": 0.07161665707826614, "num_tokens": 713280.0, "reward": -0.4185742139816284, "reward_std": 0.6441221833229065, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.8785741925239563, "rewards/length_penalty/std": 0.16615615785121918, "sampling/importance_sampling_ratio/max": 1.6265782117843628, "sampling/importance_sampling_ratio/mean": 0.9922990202903748, "sampling/importance_sampling_ratio/min": 0.634637713432312, "sampling/sampling_logp_difference/max": 0.4864785671234131, "sampling/sampling_logp_difference/mean": 0.014363318681716919, "step": 8, "step_time": 28.30564660113305 }, { "clip_ratio/high_max": 0.0009418363450095057, "clip_ratio/high_mean": 0.0009418363450095057, "clip_ratio/low_mean": 9.628144616726785e-05, "clip_ratio/low_min": 9.628144616726785e-05, "clip_ratio/region_mean": 0.0010381177882663906, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 2048.0, "completions/mean_length": 1622.97998046875, "completions/mean_terminated_length": 1503.1025390625, "completions/min_length": 873.0, "completions/min_terminated_length": 873.0, "entropy": 0.22798832952976228, "epoch": 0.024456521739130436, "frac_reward_zero_std": 0.0, "grad_norm": 0.018526818603277206, "learning_rate": 4e-05, "loss": 0.11752811819314957, "num_tokens": 796899.0, "reward": -0.1524706929922104, "reward_std": 0.5888569355010986, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732054233551, "rewards/length_penalty/mean": -0.7924706935882568, "rewards/length_penalty/std": 0.19514665007591248, "sampling/importance_sampling_ratio/max": 1.4198806285858154, "sampling/importance_sampling_ratio/mean": 0.9919716119766235, "sampling/importance_sampling_ratio/min": 0.511864960193634, "sampling/sampling_logp_difference/max": 0.6696943640708923, "sampling/sampling_logp_difference/mean": 0.015113740228116512, "step": 9, "step_time": 27.674709191778675 }, { "clip_ratio/high_max": 0.0008836857683490962, "clip_ratio/high_mean": 0.0008836857683490962, "clip_ratio/low_mean": 0.0001610520586837083, "clip_ratio/low_min": 0.0001610520586837083, "clip_ratio/region_mean": 0.0010447378270328044, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 1945.0, "completions/mean_length": 1527.5599365234375, "completions/mean_terminated_length": 1325.1666259765625, "completions/min_length": 650.0, "completions/min_terminated_length": 650.0, "entropy": 0.24462299942970275, "epoch": 0.02717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.017739074304699898, "learning_rate": 4.5e-05, "loss": 0.11460381746292114, "num_tokens": 878097.0, "reward": -0.0858789011836052, "reward_std": 0.651462733745575, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.47851812839508057, "rewards/length_penalty/mean": -0.7458789348602295, "rewards/length_penalty/std": 0.22174151241779327, "sampling/importance_sampling_ratio/max": 1.4572726488113403, "sampling/importance_sampling_ratio/mean": 0.9913933277130127, "sampling/importance_sampling_ratio/min": 0.5709614157676697, "sampling/sampling_logp_difference/max": 0.5604336261749268, "sampling/sampling_logp_difference/mean": 0.01590959168970585, "step": 10, "step_time": 27.32456138706766 }, { "clip_ratio/high_max": 0.0005791342351585627, "clip_ratio/high_mean": 0.0005791342351585627, "clip_ratio/low_mean": 0.00020766998059116304, "clip_ratio/low_min": 0.00020766998059116304, "clip_ratio/region_mean": 0.0007868041866458952, "completions/clipped_ratio": 0.5799999833106995, "completions/max_length": 2048.0, "completions/max_terminated_length": 2005.0, "completions/mean_length": 1906.219970703125, "completions/mean_terminated_length": 1710.4285888671875, "completions/min_length": 1323.0, "completions/min_terminated_length": 1323.0, "entropy": 0.2523061394691467, "epoch": 0.029891304347826088, "frac_reward_zero_std": 0.0, "grad_norm": 0.017356909811496735, "learning_rate": 5e-05, "loss": 0.074811190366745, "num_tokens": 975828.0, "reward": -0.4507714807987213, "reward_std": 0.5823943614959717, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.9307714700698853, "rewards/length_penalty/std": 0.1049875020980835, "sampling/importance_sampling_ratio/max": 1.9958995580673218, "sampling/importance_sampling_ratio/mean": 0.9913272857666016, "sampling/importance_sampling_ratio/min": 0.6561847925186157, "sampling/sampling_logp_difference/max": 0.6910948753356934, "sampling/sampling_logp_difference/mean": 0.015796277672052383, "step": 11, "step_time": 28.954787808004767 }, { "clip_ratio/high_max": 0.0004949859721818939, "clip_ratio/high_mean": 0.0004949859721818939, "clip_ratio/low_mean": 0.0001681178982835263, "clip_ratio/low_min": 0.0001681178982835263, "clip_ratio/region_mean": 0.000663103861734271, "completions/clipped_ratio": 0.4599999785423279, "completions/max_length": 2048.0, "completions/max_terminated_length": 1984.0, "completions/mean_length": 1742.43994140625, "completions/mean_terminated_length": 1482.148193359375, "completions/min_length": 974.0, "completions/min_terminated_length": 974.0, "entropy": 0.2790819376707077, "epoch": 0.03260869565217391, "frac_reward_zero_std": 0.0, "grad_norm": 0.017921727150678635, "learning_rate": 5e-05, "loss": 0.12824325263500214, "num_tokens": 1065590.0, "reward": -0.3508007824420929, "reward_std": 0.6496888995170593, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.8508007526397705, "rewards/length_penalty/std": 0.1730204075574875, "sampling/importance_sampling_ratio/max": 1.4403636455535889, "sampling/importance_sampling_ratio/mean": 0.9905098676681519, "sampling/importance_sampling_ratio/min": 0.5996456146240234, "sampling/sampling_logp_difference/max": 0.5114164352416992, "sampling/sampling_logp_difference/mean": 0.01758064143359661, "step": 12, "step_time": 27.76311965705827 }, { "clip_ratio/high_max": 0.0006719783181324601, "clip_ratio/high_mean": 0.0006719783181324601, "clip_ratio/low_mean": 0.00014902167167747394, "clip_ratio/low_min": 0.00014902167167747394, "clip_ratio/region_mean": 0.0008209999883547425, "completions/clipped_ratio": 0.4399999976158142, "completions/max_length": 2048.0, "completions/max_terminated_length": 2033.0, "completions/mean_length": 1722.1400146484375, "completions/mean_terminated_length": 1466.107177734375, "completions/min_length": 936.0, "completions/min_terminated_length": 936.0, "entropy": 0.26495963633060454, "epoch": 0.035326086956521736, "frac_reward_zero_std": 0.0, "grad_norm": 0.01803506910800934, "learning_rate": 5e-05, "loss": 0.10340610891580582, "num_tokens": 1154717.0, "reward": -0.3808886706829071, "reward_std": 0.6363792419433594, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.8408886790275574, "rewards/length_penalty/std": 0.17392964661121368, "sampling/importance_sampling_ratio/max": 1.6443123817443848, "sampling/importance_sampling_ratio/mean": 0.9907426238059998, "sampling/importance_sampling_ratio/min": 0.6476600170135498, "sampling/sampling_logp_difference/max": 0.49732232093811035, "sampling/sampling_logp_difference/mean": 0.01705038733780384, "step": 13, "step_time": 27.700532860821113 }, { "clip_ratio/high_max": 0.000790283753303811, "clip_ratio/high_mean": 0.000790283753303811, "clip_ratio/low_mean": 0.00015664642269257455, "clip_ratio/low_min": 0.00015664642269257455, "clip_ratio/region_mean": 0.0009469301789067685, "completions/clipped_ratio": 0.4399999976158142, "completions/max_length": 2048.0, "completions/max_terminated_length": 1947.0, "completions/mean_length": 1707.39990234375, "completions/mean_terminated_length": 1439.7857666015625, "completions/min_length": 939.0, "completions/min_terminated_length": 939.0, "entropy": 0.25884707272052765, "epoch": 0.03804347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.01831369660794735, "learning_rate": 5e-05, "loss": 0.10717891156673431, "num_tokens": 1242647.0, "reward": -0.2536914050579071, "reward_std": 0.6496968865394592, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.4985693693161011, "rewards/length_penalty/mean": -0.833691418170929, "rewards/length_penalty/std": 0.17602378129959106, "sampling/importance_sampling_ratio/max": 1.4137200117111206, "sampling/importance_sampling_ratio/mean": 0.9911543130874634, "sampling/importance_sampling_ratio/min": 0.6645229458808899, "sampling/sampling_logp_difference/max": 0.40868592262268066, "sampling/sampling_logp_difference/mean": 0.016555489972233772, "step": 14, "step_time": 28.091995016904548 }, { "clip_ratio/high_max": 0.0007948186947032809, "clip_ratio/high_mean": 0.0007948186947032809, "clip_ratio/low_mean": 6.400095226126723e-05, "clip_ratio/low_min": 6.400095226126723e-05, "clip_ratio/region_mean": 0.0008588196593336761, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 2024.0, "completions/mean_length": 1566.659912109375, "completions/mean_terminated_length": 1360.3714599609375, "completions/min_length": 923.0, "completions/min_terminated_length": 923.0, "entropy": 0.19630298018455505, "epoch": 0.04076086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.01908237673342228, "learning_rate": 5e-05, "loss": 0.10991621017456055, "num_tokens": 1323560.0, "reward": -0.2849707007408142, "reward_std": 0.6005774140357971, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.7649707198143005, "rewards/length_penalty/std": 0.1954003870487213, "sampling/importance_sampling_ratio/max": 1.4597245454788208, "sampling/importance_sampling_ratio/mean": 0.9932765960693359, "sampling/importance_sampling_ratio/min": 0.48890236020088196, "sampling/sampling_logp_difference/max": 0.7155925035476685, "sampling/sampling_logp_difference/mean": 0.013154168613255024, "step": 15, "step_time": 27.622011815896258 }, { "clip_ratio/high_max": 0.0007680640854232478, "clip_ratio/high_mean": 0.0007680640854232478, "clip_ratio/low_mean": 0.00010441521517350339, "clip_ratio/low_min": 0.00010441521517350339, "clip_ratio/region_mean": 0.0008724792875000276, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 2038.0, "completions/mean_length": 1529.47998046875, "completions/mean_terminated_length": 1327.8333740234375, "completions/min_length": 648.0, "completions/min_terminated_length": 648.0, "entropy": 0.2527110666036606, "epoch": 0.043478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.019366132095456123, "learning_rate": 5e-05, "loss": 0.1180194765329361, "num_tokens": 1404134.0, "reward": -0.02681640535593033, "reward_std": 0.6313645839691162, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.7468163967132568, "rewards/length_penalty/std": 0.21961458027362823, "sampling/importance_sampling_ratio/max": 1.4531008005142212, "sampling/importance_sampling_ratio/mean": 0.990881621837616, "sampling/importance_sampling_ratio/min": 0.45519256591796875, "sampling/sampling_logp_difference/max": 0.7870347499847412, "sampling/sampling_logp_difference/mean": 0.016263339668512344, "step": 16, "step_time": 26.84932530601509 }, { "clip_ratio/high_max": 0.0012978371232748032, "clip_ratio/high_mean": 0.0012978371232748032, "clip_ratio/low_mean": 4.327131027821451e-05, "clip_ratio/low_min": 4.327131027821451e-05, "clip_ratio/region_mean": 0.001341108442284167, "completions/clipped_ratio": 0.5600000023841858, "completions/max_length": 2048.0, "completions/max_terminated_length": 2007.0, "completions/mean_length": 1768.919921875, "completions/mean_terminated_length": 1413.727294921875, "completions/min_length": 865.0, "completions/min_terminated_length": 865.0, "entropy": 0.23181203305721282, "epoch": 0.04619565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.016713930293917656, "learning_rate": 5e-05, "loss": 0.08152732998132706, "num_tokens": 1495230.0, "reward": -0.4237304627895355, "reward_std": 0.6704393029212952, "rewards/correctness/mean": 0.4399999976158142, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.8637304902076721, "rewards/length_penalty/std": 0.20572726428508759, "sampling/importance_sampling_ratio/max": 1.561582088470459, "sampling/importance_sampling_ratio/mean": 0.9919421076774597, "sampling/importance_sampling_ratio/min": 0.543276309967041, "sampling/sampling_logp_difference/max": 0.6101372241973877, "sampling/sampling_logp_difference/mean": 0.015333480201661587, "step": 17, "step_time": 27.873049326473847 }, { "clip_ratio/high_max": 0.0006598436157219112, "clip_ratio/high_mean": 0.0006598436157219112, "clip_ratio/low_mean": 0.00010337726125726476, "clip_ratio/low_min": 0.00010337726125726476, "clip_ratio/region_mean": 0.0007632208871655166, "completions/clipped_ratio": 0.35999998450279236, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1746.0799560546875, "completions/mean_terminated_length": 1576.25, "completions/min_length": 1028.0, "completions/min_terminated_length": 1028.0, "entropy": 0.21664705574512483, "epoch": 0.04891304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.018009476363658905, "learning_rate": 5e-05, "loss": 0.08006907999515533, "num_tokens": 1585034.0, "reward": -0.2725781202316284, "reward_std": 0.625697135925293, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.8525781035423279, "rewards/length_penalty/std": 0.15941378474235535, "sampling/importance_sampling_ratio/max": 1.639607310295105, "sampling/importance_sampling_ratio/mean": 0.9925068020820618, "sampling/importance_sampling_ratio/min": 0.4967416226863861, "sampling/sampling_logp_difference/max": 0.6996852159500122, "sampling/sampling_logp_difference/mean": 0.013985401019454002, "step": 18, "step_time": 28.311645316891372 }, { "clip_ratio/high_max": 0.000741867849137634, "clip_ratio/high_mean": 0.000741867849137634, "clip_ratio/low_mean": 0.00018152535776607692, "clip_ratio/low_min": 0.00018152535776607692, "clip_ratio/region_mean": 0.0009233932243660093, "completions/clipped_ratio": 0.35999998450279236, "completions/max_length": 2048.0, "completions/max_terminated_length": 2011.0, "completions/mean_length": 1686.260009765625, "completions/mean_terminated_length": 1482.78125, "completions/min_length": 740.0, "completions/min_terminated_length": 740.0, "entropy": 0.2879431128501892, "epoch": 0.051630434782608696, "frac_reward_zero_std": 0.0, "grad_norm": 0.019337492063641548, "learning_rate": 5e-05, "loss": 0.08359130471944809, "num_tokens": 1672777.0, "reward": -0.18336912989616394, "reward_std": 0.6349277496337891, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.8233691453933716, "rewards/length_penalty/std": 0.19560563564300537, "sampling/importance_sampling_ratio/max": 1.4635835886001587, "sampling/importance_sampling_ratio/mean": 0.98971027135849, "sampling/importance_sampling_ratio/min": 0.635256290435791, "sampling/sampling_logp_difference/max": 0.45372676849365234, "sampling/sampling_logp_difference/mean": 0.018413560464978218, "step": 19, "step_time": 27.62987573328428 }, { "clip_ratio/high_max": 0.0006679994985461235, "clip_ratio/high_mean": 0.0006679994985461235, "clip_ratio/low_mean": 5.8021536096930505e-05, "clip_ratio/low_min": 5.8021536096930505e-05, "clip_ratio/region_mean": 0.0007260210230015218, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 2046.0, "completions/mean_length": 1787.9599609375, "completions/mean_terminated_length": 1599.6551513671875, "completions/min_length": 775.0, "completions/min_terminated_length": 775.0, "entropy": 0.2634012043476105, "epoch": 0.05434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.018007466569542885, "learning_rate": 5e-05, "loss": 0.10140140354633331, "num_tokens": 1764765.0, "reward": -0.3130273222923279, "reward_std": 0.627247154712677, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.8730273246765137, "rewards/length_penalty/std": 0.17675596475601196, "sampling/importance_sampling_ratio/max": 1.6090019941329956, "sampling/importance_sampling_ratio/mean": 0.9909850358963013, "sampling/importance_sampling_ratio/min": 0.6370789408683777, "sampling/sampling_logp_difference/max": 0.475614070892334, "sampling/sampling_logp_difference/mean": 0.01683841086924076, "step": 20, "step_time": 28.34779040887952 }, { "clip_ratio/high_max": 0.000976240006275475, "clip_ratio/high_mean": 0.000976240006275475, "clip_ratio/low_mean": 7.026160965324379e-05, "clip_ratio/low_min": 7.026160965324379e-05, "clip_ratio/region_mean": 0.00104650161229074, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 2037.0, "completions/mean_length": 1709.5399169921875, "completions/mean_terminated_length": 1564.4857177734375, "completions/min_length": 1061.0, "completions/min_terminated_length": 1061.0, "entropy": 0.20998104512691498, "epoch": 0.057065217391304345, "frac_reward_zero_std": 0.0, "grad_norm": 0.018870066851377487, "learning_rate": 5e-05, "loss": 0.08603541553020477, "num_tokens": 1853652.0, "reward": -0.1347363293170929, "reward_std": 0.5705064535140991, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.8347363471984863, "rewards/length_penalty/std": 0.15160603821277618, "sampling/importance_sampling_ratio/max": 1.4565383195877075, "sampling/importance_sampling_ratio/mean": 0.9924175143241882, "sampling/importance_sampling_ratio/min": 0.6045414805412292, "sampling/sampling_logp_difference/max": 0.5032849311828613, "sampling/sampling_logp_difference/mean": 0.01400977373123169, "step": 21, "step_time": 28.186939923092723 }, { "clip_ratio/high_max": 0.0007286557811312377, "clip_ratio/high_mean": 0.0007286557811312377, "clip_ratio/low_mean": 5.019558957428671e-05, "clip_ratio/low_min": 5.019558957428671e-05, "clip_ratio/region_mean": 0.0007788513612467796, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 2031.0, "completions/mean_length": 1543.7799072265625, "completions/mean_terminated_length": 1327.6856689453125, "completions/min_length": 758.0, "completions/min_terminated_length": 758.0, "entropy": 0.2528064101934433, "epoch": 0.059782608695652176, "frac_reward_zero_std": 0.0, "grad_norm": 0.019696656614542007, "learning_rate": 5e-05, "loss": 0.10337188839912415, "num_tokens": 1935911.0, "reward": -0.05379882827401161, "reward_std": 0.6398226022720337, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.7537988424301147, "rewards/length_penalty/std": 0.21059876680374146, "sampling/importance_sampling_ratio/max": 2.2368545532226562, "sampling/importance_sampling_ratio/mean": 0.9912877082824707, "sampling/importance_sampling_ratio/min": 0.6480669975280762, "sampling/sampling_logp_difference/max": 0.8050706386566162, "sampling/sampling_logp_difference/mean": 0.016178736463189125, "step": 22, "step_time": 27.681587701197714 }, { "clip_ratio/high_max": 0.0008125284919515252, "clip_ratio/high_mean": 0.0008125284919515252, "clip_ratio/low_mean": 0.00017936505464604123, "clip_ratio/low_min": 0.00017936505464604123, "clip_ratio/region_mean": 0.000991893548052758, "completions/clipped_ratio": 0.3400000035762787, "completions/max_length": 2048.0, "completions/max_terminated_length": 2040.0, "completions/mean_length": 1664.02001953125, "completions/mean_terminated_length": 1466.212158203125, "completions/min_length": 824.0, "completions/min_terminated_length": 824.0, "entropy": 0.27761187553405764, "epoch": 0.0625, "frac_reward_zero_std": 0.0, "grad_norm": 0.021949486806988716, "learning_rate": 5e-05, "loss": 0.11840050667524338, "num_tokens": 2025232.0, "reward": -0.2525097727775574, "reward_std": 0.6308436393737793, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.8125097751617432, "rewards/length_penalty/std": 0.18829484283924103, "sampling/importance_sampling_ratio/max": 1.6358195543289185, "sampling/importance_sampling_ratio/mean": 0.9903144836425781, "sampling/importance_sampling_ratio/min": 0.5315372943878174, "sampling/sampling_logp_difference/max": 0.6319819688796997, "sampling/sampling_logp_difference/mean": 0.017899619415402412, "step": 23, "step_time": 28.560938307084143 }, { "clip_ratio/high_max": 0.0005096927285194397, "clip_ratio/high_mean": 0.0005096927285194397, "clip_ratio/low_mean": 8.583837043261156e-05, "clip_ratio/low_min": 8.583837043261156e-05, "clip_ratio/region_mean": 0.0005955311004072428, "completions/clipped_ratio": 0.4399999976158142, "completions/max_length": 2048.0, "completions/max_terminated_length": 2042.0, "completions/mean_length": 1590.5599365234375, "completions/mean_terminated_length": 1231.1429443359375, "completions/min_length": 672.0, "completions/min_terminated_length": 672.0, "entropy": 0.23703448474407196, "epoch": 0.06521739130434782, "frac_reward_zero_std": 0.0, "grad_norm": 0.019747884944081306, "learning_rate": 5e-05, "loss": 0.09910769760608673, "num_tokens": 2107050.0, "reward": -0.2166406214237213, "reward_std": 0.7164656519889832, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.7766406536102295, "rewards/length_penalty/std": 0.24764245748519897, "sampling/importance_sampling_ratio/max": 1.457215428352356, "sampling/importance_sampling_ratio/mean": 0.9916846752166748, "sampling/importance_sampling_ratio/min": 0.5029337406158447, "sampling/sampling_logp_difference/max": 0.6872968673706055, "sampling/sampling_logp_difference/mean": 0.015448657795786858, "step": 24, "step_time": 27.499916886212304 }, { "clip_ratio/high_max": 0.0008482657605782152, "clip_ratio/high_mean": 0.0008482657605782152, "clip_ratio/low_mean": 0.00022012473782524468, "clip_ratio/low_min": 0.00022012473782524468, "clip_ratio/region_mean": 0.0010683904984034598, "completions/clipped_ratio": 0.23999999463558197, "completions/max_length": 2048.0, "completions/max_terminated_length": 1919.0, "completions/mean_length": 1442.97998046875, "completions/mean_terminated_length": 1251.9210205078125, "completions/min_length": 633.0, "completions/min_terminated_length": 633.0, "entropy": 0.2630555659532547, "epoch": 0.06793478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.021064141765236855, "learning_rate": 5e-05, "loss": 0.11863033473491669, "num_tokens": 2181599.0, "reward": 0.055419921875, "reward_std": 0.6141738295555115, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.7045800685882568, "rewards/length_penalty/std": 0.21538712084293365, "sampling/importance_sampling_ratio/max": 1.5541377067565918, "sampling/importance_sampling_ratio/mean": 0.990498960018158, "sampling/importance_sampling_ratio/min": 0.40694311261177063, "sampling/sampling_logp_difference/max": 0.8990819454193115, "sampling/sampling_logp_difference/mean": 0.017718778923153877, "step": 25, "step_time": 26.41467908094637 }, { "clip_ratio/high_max": 0.000819263607263565, "clip_ratio/high_mean": 0.000819263607263565, "clip_ratio/low_mean": 0.00011166772019350901, "clip_ratio/low_min": 0.00011166772019350901, "clip_ratio/region_mean": 0.0009309313609264791, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 1262.239990234375, "completions/mean_terminated_length": 1246.2041015625, "completions/min_length": 687.0, "completions/min_terminated_length": 687.0, "entropy": 0.21845731139183044, "epoch": 0.07065217391304347, "frac_reward_zero_std": 0.0, "grad_norm": 0.019048262387514114, "learning_rate": 5e-05, "loss": 0.11241159588098526, "num_tokens": 2247181.0, "reward": -0.056328125298023224, "reward_std": 0.48785343766212463, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.6163281202316284, "rewards/length_penalty/std": 0.16271910071372986, "sampling/importance_sampling_ratio/max": 1.8472636938095093, "sampling/importance_sampling_ratio/mean": 0.9921180605888367, "sampling/importance_sampling_ratio/min": 0.45372647047042847, "sampling/sampling_logp_difference/max": 0.7902607917785645, "sampling/sampling_logp_difference/mean": 0.015436006709933281, "step": 26, "step_time": 26.193258133949712 }, { "clip_ratio/high_max": 0.00045611696768901313, "clip_ratio/high_mean": 0.00045611696768901313, "clip_ratio/low_mean": 0.00010173475166084245, "clip_ratio/low_min": 0.00010173475166084245, "clip_ratio/region_mean": 0.0005578517040703446, "completions/clipped_ratio": 0.3999999761581421, "completions/max_length": 2048.0, "completions/max_terminated_length": 2031.0, "completions/mean_length": 1539.419921875, "completions/mean_terminated_length": 1200.36669921875, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.2073049932718277, "epoch": 0.07336956521739131, "frac_reward_zero_std": 0.0, "grad_norm": 0.0178088266402483, "learning_rate": 5e-05, "loss": 0.05440214276313782, "num_tokens": 2326402.0, "reward": -0.1516699194908142, "reward_std": 0.7180583477020264, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.7516699433326721, "rewards/length_penalty/std": 0.26075154542922974, "sampling/importance_sampling_ratio/max": 2.0076329708099365, "sampling/importance_sampling_ratio/mean": 0.9929074645042419, "sampling/importance_sampling_ratio/min": 0.3890666961669922, "sampling/sampling_logp_difference/max": 0.9440045356750488, "sampling/sampling_logp_difference/mean": 0.014081662520766258, "step": 27, "step_time": 26.73872535675764 }, { "clip_ratio/high_max": 0.0007280473771970719, "clip_ratio/high_mean": 0.0007280473771970719, "clip_ratio/low_mean": 6.700347003061325e-05, "clip_ratio/low_min": 6.700347003061325e-05, "clip_ratio/region_mean": 0.0007950508443173021, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1852.0, "completions/mean_length": 1207.4599609375, "completions/mean_terminated_length": 1190.30615234375, "completions/min_length": 638.0, "completions/min_terminated_length": 638.0, "entropy": 0.1798260122537613, "epoch": 0.07608695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.016198398545384407, "learning_rate": 5e-05, "loss": 0.09480622410774231, "num_tokens": 2390135.0, "reward": 0.2104199230670929, "reward_std": 0.4322068691253662, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.5895800590515137, "rewards/length_penalty/std": 0.1593828648328781, "sampling/importance_sampling_ratio/max": 1.6265084743499756, "sampling/importance_sampling_ratio/mean": 0.993723452091217, "sampling/importance_sampling_ratio/min": 0.38461166620254517, "sampling/sampling_logp_difference/max": 0.9555211067199707, "sampling/sampling_logp_difference/mean": 0.012959480285644531, "step": 28, "step_time": 25.803300652187318 }, { "clip_ratio/high_max": 0.0007129237637855113, "clip_ratio/high_mean": 0.0007129237637855113, "clip_ratio/low_mean": 0.00013153162435628474, "clip_ratio/low_min": 0.00013153162435628474, "clip_ratio/region_mean": 0.000844455393962562, "completions/clipped_ratio": 0.3799999952316284, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 1605.52001953125, "completions/mean_terminated_length": 1334.322509765625, "completions/min_length": 545.0, "completions/min_terminated_length": 545.0, "entropy": 0.23342860341072083, "epoch": 0.07880434782608696, "frac_reward_zero_std": 0.0, "grad_norm": 0.019112825393676758, "learning_rate": 5e-05, "loss": 0.08988890796899796, "num_tokens": 2473851.0, "reward": -0.08394531160593033, "reward_std": 0.643122673034668, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.7839453220367432, "rewards/length_penalty/std": 0.2589260935783386, "sampling/importance_sampling_ratio/max": 2.976001739501953, "sampling/importance_sampling_ratio/mean": 0.9917635917663574, "sampling/importance_sampling_ratio/min": 0.3071954846382141, "sampling/sampling_logp_difference/max": 1.180271029472351, "sampling/sampling_logp_difference/mean": 0.016178693622350693, "step": 29, "step_time": 27.464959647972137 }, { "clip_ratio/high_max": 0.0007170933298766613, "clip_ratio/high_mean": 0.0007170933298766613, "clip_ratio/low_mean": 7.078182243276387e-05, "clip_ratio/low_min": 7.078182243276387e-05, "clip_ratio/region_mean": 0.0007878751552198082, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1948.0, "completions/mean_length": 1215.739990234375, "completions/mean_terminated_length": 1102.25, "completions/min_length": 676.0, "completions/min_terminated_length": 676.0, "entropy": 0.1600349336862564, "epoch": 0.08152173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.01690404675900936, "learning_rate": 5e-05, "loss": 0.10917580127716064, "num_tokens": 2536428.0, "reward": 0.286376953125, "reward_std": 0.49757134914398193, "rewards/correctness/mean": 0.8799999952316284, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.5936230421066284, "rewards/length_penalty/std": 0.20075169205665588, "sampling/importance_sampling_ratio/max": 2.474348306655884, "sampling/importance_sampling_ratio/mean": 0.9940484762191772, "sampling/importance_sampling_ratio/min": 0.2940029501914978, "sampling/sampling_logp_difference/max": 1.224165439605713, "sampling/sampling_logp_difference/mean": 0.012239653617143631, "step": 30, "step_time": 25.22584995208308 }, { "clip_ratio/high_max": 0.000856519874650985, "clip_ratio/high_mean": 0.000856519874650985, "clip_ratio/low_mean": 8.000027883099392e-05, "clip_ratio/low_min": 8.000027883099392e-05, "clip_ratio/region_mean": 0.0009365201694890857, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1861.0, "completions/mean_length": 1463.8800048828125, "completions/mean_terminated_length": 1317.8499755859375, "completions/min_length": 510.0, "completions/min_terminated_length": 510.0, "entropy": 0.17065878212451935, "epoch": 0.08423913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.01730084978044033, "learning_rate": 5e-05, "loss": 0.12122836709022522, "num_tokens": 2612752.0, "reward": -0.29478514194488525, "reward_std": 0.5411331653594971, "rewards/correctness/mean": 0.41999998688697815, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.7147851586341858, "rewards/length_penalty/std": 0.20178328454494476, "sampling/importance_sampling_ratio/max": 2.579355478286743, "sampling/importance_sampling_ratio/mean": 0.9938713908195496, "sampling/importance_sampling_ratio/min": 0.2596469223499298, "sampling/sampling_logp_difference/max": 1.3484325408935547, "sampling/sampling_logp_difference/mean": 0.012738464400172234, "step": 31, "step_time": 27.176139670424163 }, { "clip_ratio/high_max": 0.0008451115863863379, "clip_ratio/high_mean": 0.0008451115863863379, "clip_ratio/low_mean": 0.00011503964924486354, "clip_ratio/low_min": 0.00011503964924486354, "clip_ratio/region_mean": 0.0009601512225344777, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 2000.0, "completions/mean_length": 1191.3399658203125, "completions/mean_terminated_length": 1028.1666259765625, "completions/min_length": 552.0, "completions/min_terminated_length": 552.0, "entropy": 0.21862303614616393, "epoch": 0.08695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.01855313777923584, "learning_rate": 5e-05, "loss": 0.1444239616394043, "num_tokens": 2674249.0, "reward": 0.23829101026058197, "reward_std": 0.5837259888648987, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.5817089676856995, "rewards/length_penalty/std": 0.2382737696170807, "sampling/importance_sampling_ratio/max": 1.9388597011566162, "sampling/importance_sampling_ratio/mean": 0.9922151565551758, "sampling/importance_sampling_ratio/min": 0.24645483493804932, "sampling/sampling_logp_difference/max": 1.4005765914916992, "sampling/sampling_logp_difference/mean": 0.0162365585565567, "step": 32, "step_time": 25.234165051020682 }, { "clip_ratio/high_max": 0.0009740249835886062, "clip_ratio/high_mean": 0.0009740249835886062, "clip_ratio/low_mean": 5.943443829892203e-05, "clip_ratio/low_min": 5.943443829892203e-05, "clip_ratio/region_mean": 0.001033459440805018, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1974.0, "completions/mean_length": 1382.0599365234375, "completions/mean_terminated_length": 1148.0810546875, "completions/min_length": 706.0, "completions/min_terminated_length": 706.0, "entropy": 0.20403595566749572, "epoch": 0.08967391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.016193265095353127, "learning_rate": 5e-05, "loss": 0.05690198391675949, "num_tokens": 2746992.0, "reward": -0.09483398497104645, "reward_std": 0.6409366726875305, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.6748340129852295, "rewards/length_penalty/std": 0.2429748922586441, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9926742315292358, "sampling/importance_sampling_ratio/min": 0.1798250824213028, "sampling/sampling_logp_difference/max": 1.7157707214355469, "sampling/sampling_logp_difference/mean": 0.014851015992462635, "step": 33, "step_time": 26.865770722040907 }, { "clip_ratio/high_max": 0.0012180538265965878, "clip_ratio/high_mean": 0.0012180538265965878, "clip_ratio/low_mean": 2.9444240499287844e-05, "clip_ratio/low_min": 2.9444240499287844e-05, "clip_ratio/region_mean": 0.0012474980554543435, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 1734.0, "completions/mean_length": 1317.179931640625, "completions/mean_terminated_length": 1111.05126953125, "completions/min_length": 647.0, "completions/min_terminated_length": 647.0, "entropy": 0.22754225134849548, "epoch": 0.09239130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.020097807049751282, "learning_rate": 5e-05, "loss": 0.0710531622171402, "num_tokens": 2816741.0, "reward": 0.11684569716453552, "reward_std": 0.61951744556427, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.6431543231010437, "rewards/length_penalty/std": 0.2254319041967392, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9915940761566162, "sampling/importance_sampling_ratio/min": 0.23836448788642883, "sampling/sampling_logp_difference/max": 1.4339543581008911, "sampling/sampling_logp_difference/mean": 0.01691325753927231, "step": 34, "step_time": 26.204810510622337 }, { "clip_ratio/high_max": 0.0008569001860450953, "clip_ratio/high_mean": 0.0008569001860450953, "clip_ratio/low_mean": 0.00013024549843976274, "clip_ratio/low_min": 0.00013024549843976274, "clip_ratio/region_mean": 0.0009871456888504327, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 2030.0, "completions/mean_length": 1345.1400146484375, "completions/mean_terminated_length": 1211.261962890625, "completions/min_length": 632.0, "completions/min_terminated_length": 632.0, "entropy": 0.26689461171627044, "epoch": 0.09510869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.0222671777009964, "learning_rate": 5e-05, "loss": 0.15181683003902435, "num_tokens": 2886728.0, "reward": 0.1431933492422104, "reward_std": 0.5873222351074219, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.6568066477775574, "rewards/length_penalty/std": 0.22363705933094025, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.990323543548584, "sampling/importance_sampling_ratio/min": 0.16977547109127045, "sampling/sampling_logp_difference/max": 1.7732784748077393, "sampling/sampling_logp_difference/mean": 0.01977042853832245, "step": 35, "step_time": 25.99194298312068 }, { "clip_ratio/high_max": 0.0007262625789735466, "clip_ratio/high_mean": 0.0007262625789735466, "clip_ratio/low_mean": 0.0001643200113903731, "clip_ratio/low_min": 0.0001643200113903731, "clip_ratio/region_mean": 0.0008905825903639198, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1975.0, "completions/mean_length": 1117.199951171875, "completions/mean_terminated_length": 990.2727661132812, "completions/min_length": 532.0, "completions/min_terminated_length": 532.0, "entropy": 0.1896991550922394, "epoch": 0.09782608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.01747210882604122, "learning_rate": 5e-05, "loss": 0.058442991226911545, "num_tokens": 2945058.0, "reward": 0.33449217677116394, "reward_std": 0.5202338695526123, "rewards/correctness/mean": 0.8799999952316284, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.5455077886581421, "rewards/length_penalty/std": 0.22712095081806183, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930076599121094, "sampling/importance_sampling_ratio/min": 0.1576518714427948, "sampling/sampling_logp_difference/max": 1.8473659753799438, "sampling/sampling_logp_difference/mean": 0.01649167202413082, "step": 36, "step_time": 25.64875900489278 }, { "clip_ratio/high_max": 0.0007515452452935278, "clip_ratio/high_mean": 0.0007515452452935278, "clip_ratio/low_mean": 0.00017000650841509924, "clip_ratio/low_min": 0.00017000650841509924, "clip_ratio/region_mean": 0.0009215517551638186, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 1888.0, "completions/mean_length": 1225.93994140625, "completions/mean_terminated_length": 994.0769653320312, "completions/min_length": 605.0, "completions/min_terminated_length": 605.0, "entropy": 0.2357550859451294, "epoch": 0.10054347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.019464492797851562, "learning_rate": 5e-05, "loss": 0.09758821129798889, "num_tokens": 3009875.0, "reward": -0.01860351487994194, "reward_std": 0.6543649435043335, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.5986034870147705, "rewards/length_penalty/std": 0.24733325839042664, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9915767908096313, "sampling/importance_sampling_ratio/min": 0.23241811990737915, "sampling/sampling_logp_difference/max": 1.4991412162780762, "sampling/sampling_logp_difference/mean": 0.01839047111570835, "step": 37, "step_time": 26.028307612985373 }, { "clip_ratio/high_max": 0.0007471224380424246, "clip_ratio/high_mean": 0.0007471224380424246, "clip_ratio/low_mean": 5.083655851194635e-05, "clip_ratio/low_min": 5.083655851194635e-05, "clip_ratio/region_mean": 0.0007979590183822439, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1990.0, "completions/mean_length": 1287.699951171875, "completions/mean_terminated_length": 1203.2222900390625, "completions/min_length": 667.0, "completions/min_terminated_length": 667.0, "entropy": 0.1729876458644867, "epoch": 0.10326086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.01705159805715084, "learning_rate": 5e-05, "loss": 0.07221940904855728, "num_tokens": 3077650.0, "reward": 0.25124022364616394, "reward_std": 0.4530826508998871, "rewards/correctness/mean": 0.8799999952316284, "rewards/correctness/std": 0.32826074957847595, "rewards/length_penalty/mean": -0.6287597417831421, "rewards/length_penalty/std": 0.2424236387014389, "sampling/importance_sampling_ratio/max": 2.784482002258301, "sampling/importance_sampling_ratio/mean": 0.9937966465950012, "sampling/importance_sampling_ratio/min": 0.0835733637213707, "sampling/sampling_logp_difference/max": 2.4820303916931152, "sampling/sampling_logp_difference/mean": 0.014295706525444984, "step": 38, "step_time": 26.3849402081687 }, { "clip_ratio/high_max": 0.00047908292326610536, "clip_ratio/high_mean": 0.00047908292326610536, "clip_ratio/low_mean": 6.898379942867904e-05, "clip_ratio/low_min": 6.898379942867904e-05, "clip_ratio/region_mean": 0.0005480667285155505, "completions/clipped_ratio": 0.17999999225139618, "completions/max_length": 2048.0, "completions/max_terminated_length": 1733.0, "completions/mean_length": 1131.1400146484375, "completions/mean_terminated_length": 929.8779907226562, "completions/min_length": 473.0, "completions/min_terminated_length": 473.0, "entropy": 0.18207353055477143, "epoch": 0.10597826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.02022084966301918, "learning_rate": 5e-05, "loss": 0.05686383694410324, "num_tokens": 3136697.0, "reward": 0.06768554449081421, "reward_std": 0.7044277191162109, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.5523144602775574, "rewards/length_penalty/std": 0.25435444712638855, "sampling/importance_sampling_ratio/max": 2.187497615814209, "sampling/importance_sampling_ratio/mean": 0.993350088596344, "sampling/importance_sampling_ratio/min": 0.27275171875953674, "sampling/sampling_logp_difference/max": 1.2991933822631836, "sampling/sampling_logp_difference/mean": 0.014788741245865822, "step": 39, "step_time": 24.97533990885131 }, { "clip_ratio/high_max": 0.0007706037955358625, "clip_ratio/high_mean": 0.0007706037955358625, "clip_ratio/low_mean": 9.303098486270755e-05, "clip_ratio/low_min": 9.303098486270755e-05, "clip_ratio/region_mean": 0.0008636347949504853, "completions/clipped_ratio": 0.0, "completions/max_length": 1812.0, "completions/max_terminated_length": 1812.0, "completions/mean_length": 870.8999633789062, "completions/mean_terminated_length": 870.8999633789062, "completions/min_length": 654.0, "completions/min_terminated_length": 654.0, "entropy": 0.1895820140838623, "epoch": 0.10869565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.014811172150075436, "learning_rate": 5e-05, "loss": 0.06211411952972412, "num_tokens": 3182962.0, "reward": 0.574755847454071, "reward_std": 0.1076437383890152, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.42524415254592896, "rewards/length_penalty/std": 0.1076437383890152, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.993091881275177, "sampling/importance_sampling_ratio/min": 0.11348514258861542, "sampling/sampling_logp_difference/max": 2.1760833263397217, "sampling/sampling_logp_difference/mean": 0.017822640016674995, "step": 40, "step_time": 21.239220429910347 }, { "clip_ratio/high_max": 0.000729369098553434, "clip_ratio/high_mean": 0.000729369098553434, "clip_ratio/low_mean": 6.442431767936796e-05, "clip_ratio/low_min": 6.442431767936796e-05, "clip_ratio/region_mean": 0.0007937934075016529, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1935.0, "completions/mean_length": 1214.1199951171875, "completions/mean_terminated_length": 1100.4091796875, "completions/min_length": 598.0, "completions/min_terminated_length": 598.0, "entropy": 0.20386375486850739, "epoch": 0.11141304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.0198789294809103, "learning_rate": 5e-05, "loss": 0.09383414685726166, "num_tokens": 3247408.0, "reward": -0.05283202975988388, "reward_std": 0.612436830997467, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.5928320288658142, "rewards/length_penalty/std": 0.20813514292240143, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9923471212387085, "sampling/importance_sampling_ratio/min": 0.11420464515686035, "sampling/sampling_logp_difference/max": 2.1697633266448975, "sampling/sampling_logp_difference/mean": 0.017043285071849823, "step": 41, "step_time": 25.912487969035283 }, { "clip_ratio/high_max": 0.0005029736028518527, "clip_ratio/high_mean": 0.0005029736028518527, "clip_ratio/low_mean": 0.00012810062034986913, "clip_ratio/low_min": 0.00012810062034986913, "clip_ratio/region_mean": 0.0006310742348432541, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1992.0, "completions/mean_length": 1178.3599853515625, "completions/mean_terminated_length": 960.9500122070312, "completions/min_length": 471.0, "completions/min_terminated_length": 471.0, "entropy": 0.20278692245483398, "epoch": 0.11413043478260869, "frac_reward_zero_std": 0.0, "grad_norm": 0.01870260015130043, "learning_rate": 5e-05, "loss": 0.03833295777440071, "num_tokens": 3309316.0, "reward": 0.22462889552116394, "reward_std": 0.6349654197692871, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5753710865974426, "rewards/length_penalty/std": 0.2908514440059662, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9925538301467896, "sampling/importance_sampling_ratio/min": 0.16318759322166443, "sampling/sampling_logp_difference/max": 1.8128548860549927, "sampling/sampling_logp_difference/mean": 0.01731746457517147, "step": 42, "step_time": 26.0047237186227 }, { "clip_ratio/high_max": 0.001067713089287281, "clip_ratio/high_mean": 0.001067713089287281, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001067713089287281, "completions/clipped_ratio": 0.0, "completions/max_length": 1500.0, "completions/max_terminated_length": 1500.0, "completions/mean_length": 777.4400024414062, "completions/mean_terminated_length": 777.4400024414062, "completions/min_length": 481.0, "completions/min_terminated_length": 481.0, "entropy": 0.2059490293264389, "epoch": 0.11684782608695653, "frac_reward_zero_std": 0.0, "grad_norm": 0.01624198630452156, "learning_rate": 5e-05, "loss": 0.05506262555718422, "num_tokens": 3350158.0, "reward": 0.600390613079071, "reward_std": 0.18859446048736572, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.3796093761920929, "rewards/length_penalty/std": 0.12124241143465042, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9921716451644897, "sampling/importance_sampling_ratio/min": 0.048016149550676346, "sampling/sampling_logp_difference/max": 3.0362179279327393, "sampling/sampling_logp_difference/mean": 0.021691124886274338, "step": 43, "step_time": 17.796076989499852 }, { "clip_ratio/high_max": 0.0006964510248508304, "clip_ratio/high_mean": 0.0006964510248508304, "clip_ratio/low_mean": 0.00017296440637437627, "clip_ratio/low_min": 0.00017296440637437627, "clip_ratio/region_mean": 0.0008694154501426965, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1803.0, "completions/mean_length": 1044.780029296875, "completions/mean_terminated_length": 1024.30615234375, "completions/min_length": 441.0, "completions/min_terminated_length": 441.0, "entropy": 0.1765652060508728, "epoch": 0.11956521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.015211697667837143, "learning_rate": 5e-05, "loss": 0.09405821561813354, "num_tokens": 3405397.0, "reward": 0.4698534905910492, "reward_std": 0.28211453557014465, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.5101464986801147, "rewards/length_penalty/std": 0.19898390769958496, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9933960437774658, "sampling/importance_sampling_ratio/min": 0.05977986007928848, "sampling/sampling_logp_difference/max": 2.8170864582061768, "sampling/sampling_logp_difference/mean": 0.016818420961499214, "step": 44, "step_time": 24.691332526504993 }, { "clip_ratio/high_max": 0.0007825112727005035, "clip_ratio/high_mean": 0.0007825112727005035, "clip_ratio/low_mean": 3.903874894604087e-05, "clip_ratio/low_min": 3.903874894604087e-05, "clip_ratio/region_mean": 0.0008215500216465444, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1795.0, "completions/mean_length": 1010.0199584960938, "completions/mean_terminated_length": 966.7708740234375, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "entropy": 0.2038260132074356, "epoch": 0.12228260869565218, "frac_reward_zero_std": 0.0, "grad_norm": 0.016963917762041092, "learning_rate": 5e-05, "loss": 0.05373412370681763, "num_tokens": 3458838.0, "reward": 0.28682616353034973, "reward_std": 0.5763336420059204, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.49317383766174316, "rewards/length_penalty/std": 0.2312302142381668, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9924213886260986, "sampling/importance_sampling_ratio/min": 0.06157507002353668, "sampling/sampling_logp_difference/max": 2.7874982357025146, "sampling/sampling_logp_difference/mean": 0.018136093392968178, "step": 45, "step_time": 25.164567972067744 }, { "clip_ratio/high_max": 0.00039000823890091854, "clip_ratio/high_mean": 0.00039000823890091854, "clip_ratio/low_mean": 7.269879861269147e-05, "clip_ratio/low_min": 7.269879861269147e-05, "clip_ratio/region_mean": 0.0004627070476999506, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1188.0, "completions/mean_length": 1086.6400146484375, "completions/mean_terminated_length": 846.2999877929688, "completions/min_length": 463.0, "completions/min_terminated_length": 463.0, "entropy": 0.22776823937892915, "epoch": 0.125, "frac_reward_zero_std": 0.0, "grad_norm": 0.015369636937975883, "learning_rate": 5e-05, "loss": 0.04503121227025986, "num_tokens": 3516680.0, "reward": 0.2694140672683716, "reward_std": 0.6474911570549011, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5305859446525574, "rewards/length_penalty/std": 0.25373587012290955, "sampling/importance_sampling_ratio/max": 2.9789910316467285, "sampling/importance_sampling_ratio/mean": 0.9916025996208191, "sampling/importance_sampling_ratio/min": 0.04188171774148941, "sampling/sampling_logp_difference/max": 3.172905921936035, "sampling/sampling_logp_difference/mean": 0.019508102908730507, "step": 46, "step_time": 25.130053090164438 }, { "clip_ratio/high_max": 0.0006507517013233155, "clip_ratio/high_mean": 0.0006507517013233155, "clip_ratio/low_mean": 0.00011765561066567898, "clip_ratio/low_min": 0.00011765561066567898, "clip_ratio/region_mean": 0.0007684073178097605, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1485.0, "completions/mean_length": 826.6199951171875, "completions/mean_terminated_length": 775.7291870117188, "completions/min_length": 500.0, "completions/min_terminated_length": 500.0, "entropy": 0.175468310713768, "epoch": 0.12771739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.01958637870848179, "learning_rate": 5e-05, "loss": 0.06920386850833893, "num_tokens": 3560401.0, "reward": 0.5163769125938416, "reward_std": 0.41254669427871704, "rewards/correctness/mean": 0.9200000166893005, "rewards/correctness/std": 0.27404752373695374, "rewards/length_penalty/mean": -0.4036230444908142, "rewards/length_penalty/std": 0.16869166493415833, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.993443489074707, "sampling/importance_sampling_ratio/min": 0.13633184134960175, "sampling/sampling_logp_difference/max": 1.9926633834838867, "sampling/sampling_logp_difference/mean": 0.01789533719420433, "step": 47, "step_time": 23.82672528317198 }, { "clip_ratio/high_max": 0.0005032541987020523, "clip_ratio/high_mean": 0.0005032541987020523, "clip_ratio/low_mean": 6.262429669732228e-05, "clip_ratio/low_min": 6.262429669732228e-05, "clip_ratio/region_mean": 0.0005658784910337999, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 1960.0, "completions/mean_length": 1017.1199951171875, "completions/mean_terminated_length": 927.478271484375, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "entropy": 0.1553356945514679, "epoch": 0.13043478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.017065836116671562, "learning_rate": 5e-05, "loss": 0.05763271450996399, "num_tokens": 3614217.0, "reward": 0.18335936963558197, "reward_std": 0.617246687412262, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.4966406226158142, "rewards/length_penalty/std": 0.2483815848827362, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9945675134658813, "sampling/importance_sampling_ratio/min": 0.1440926343202591, "sampling/sampling_logp_difference/max": 2.313066005706787, "sampling/sampling_logp_difference/mean": 0.014977255836129189, "step": 48, "step_time": 25.213895262451842 }, { "clip_ratio/high_max": 0.0016091061057522892, "clip_ratio/high_mean": 0.0016091061057522892, "clip_ratio/low_mean": 4.549590521492064e-05, "clip_ratio/low_min": 4.549590521492064e-05, "clip_ratio/region_mean": 0.0016546020051464438, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1164.0, "completions/mean_length": 972.0799560546875, "completions/mean_terminated_length": 703.1000366210938, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.24074310958385467, "epoch": 0.1331521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.01453313510864973, "learning_rate": 5e-05, "loss": 0.04727436602115631, "num_tokens": 3665511.0, "reward": 0.3253515660762787, "reward_std": 0.6757630109786987, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.47464844584465027, "rewards/length_penalty/std": 0.2810002267360687, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.991299033164978, "sampling/importance_sampling_ratio/min": 0.12447156012058258, "sampling/sampling_logp_difference/max": 2.0836780071258545, "sampling/sampling_logp_difference/mean": 0.021517034620046616, "step": 49, "step_time": 24.520182807929814 }, { "clip_ratio/high_max": 0.0003247870015911758, "clip_ratio/high_mean": 0.0003247870015911758, "clip_ratio/low_mean": 0.00028453292179619893, "clip_ratio/low_min": 0.00028453292179619893, "clip_ratio/region_mean": 0.0006093199306633323, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 912.2799682617188, "completions/mean_terminated_length": 628.3500366210938, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.25229776501655576, "epoch": 0.1358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.013235251419246197, "learning_rate": 5e-05, "loss": 0.02668197639286518, "num_tokens": 3713535.0, "reward": 0.15455077588558197, "reward_std": 0.7314342856407166, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.44544923305511475, "rewards/length_penalty/std": 0.29221686720848083, "sampling/importance_sampling_ratio/max": 2.721278667449951, "sampling/importance_sampling_ratio/mean": 0.9899016618728638, "sampling/importance_sampling_ratio/min": 0.042893968522548676, "sampling/sampling_logp_difference/max": 3.14902400970459, "sampling/sampling_logp_difference/mean": 0.022883890196681023, "step": 50, "step_time": 24.369463649811223 }, { "clip_ratio/high_max": 0.000643822574056685, "clip_ratio/high_mean": 0.000643822574056685, "clip_ratio/low_mean": 6.33914431091398e-05, "clip_ratio/low_min": 6.33914431091398e-05, "clip_ratio/region_mean": 0.0007072140229865909, "completions/clipped_ratio": 0.0, "completions/max_length": 1970.0, "completions/max_terminated_length": 1970.0, "completions/mean_length": 766.9599609375, "completions/mean_terminated_length": 766.9599609375, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "entropy": 0.16464298963546753, "epoch": 0.13858695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.015319639816880226, "learning_rate": 5e-05, "loss": 0.06438955664634705, "num_tokens": 3754393.0, "reward": 0.6255077719688416, "reward_std": 0.1482277661561966, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.37449219822883606, "rewards/length_penalty/std": 0.1482277661561966, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9936192035675049, "sampling/importance_sampling_ratio/min": 0.11222824454307556, "sampling/sampling_logp_difference/max": 2.187220573425293, "sampling/sampling_logp_difference/mean": 0.018299877643585205, "step": 51, "step_time": 22.948548633838072 }, { "clip_ratio/high_max": 0.0007740166562143713, "clip_ratio/high_mean": 0.0007740166562143713, "clip_ratio/low_mean": 0.0001623515330720693, "clip_ratio/low_min": 0.0001623515330720693, "clip_ratio/region_mean": 0.0009363682067487389, "completions/clipped_ratio": 0.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 700.0799560546875, "completions/mean_terminated_length": 700.0799560546875, "completions/min_length": 454.0, "completions/min_terminated_length": 454.0, "entropy": 0.15744704604148865, "epoch": 0.14130434782608695, "frac_reward_zero_std": 0.0, "grad_norm": 0.012992658652365208, "learning_rate": 5e-05, "loss": 0.04073449596762657, "num_tokens": 3791657.0, "reward": 0.4581640660762787, "reward_std": 0.41504931449890137, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.34183594584465027, "rewards/length_penalty/std": 0.11200068145990372, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9942061305046082, "sampling/importance_sampling_ratio/min": 0.15024611353874207, "sampling/sampling_logp_difference/max": 1.8954805135726929, "sampling/sampling_logp_difference/mean": 0.019554542377591133, "step": 52, "step_time": 16.728716250276193 }, { "clip_ratio/high_max": 0.0006250745645957068, "clip_ratio/high_mean": 0.0006250745645957068, "clip_ratio/low_mean": 2.658160519786179e-05, "clip_ratio/low_min": 2.658160519786179e-05, "clip_ratio/region_mean": 0.0006516561697935686, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1354.0, "completions/mean_length": 807.3599853515625, "completions/mean_terminated_length": 755.6666870117188, "completions/min_length": 397.0, "completions/min_terminated_length": 397.0, "entropy": 0.1829791933298111, "epoch": 0.14402173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.021517958492040634, "learning_rate": 5e-05, "loss": 0.10340322554111481, "num_tokens": 3834745.0, "reward": 0.3657812476158142, "reward_std": 0.571151077747345, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.3942187428474426, "rewards/length_penalty/std": 0.1765245497226715, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9934973120689392, "sampling/importance_sampling_ratio/min": 0.05403505638241768, "sampling/sampling_logp_difference/max": 2.9181222915649414, "sampling/sampling_logp_difference/mean": 0.021358292549848557, "step": 53, "step_time": 23.810034946305677 }, { "clip_ratio/high_max": 0.0007082508644089102, "clip_ratio/high_mean": 0.0007082508644089102, "clip_ratio/low_mean": 0.00010528063430683687, "clip_ratio/low_min": 0.00010528063430683687, "clip_ratio/region_mean": 0.0008135314856190234, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1746.0, "completions/mean_length": 1111.3199462890625, "completions/mean_terminated_length": 983.5909423828125, "completions/min_length": 512.0, "completions/min_terminated_length": 512.0, "entropy": 0.1832393229007721, "epoch": 0.14673913043478262, "frac_reward_zero_std": 0.0, "grad_norm": 0.019397616386413574, "learning_rate": 5e-05, "loss": 0.11991956830024719, "num_tokens": 3894131.0, "reward": 0.31736326217651367, "reward_std": 0.5375475883483887, "rewards/correctness/mean": 0.8600000143051147, "rewards/correctness/std": 0.3505098223686218, "rewards/length_penalty/mean": -0.5426366925239563, "rewards/length_penalty/std": 0.23527614772319794, "sampling/importance_sampling_ratio/max": 2.2643704414367676, "sampling/importance_sampling_ratio/mean": 0.9930412769317627, "sampling/importance_sampling_ratio/min": 0.04445350170135498, "sampling/sampling_logp_difference/max": 3.113311529159546, "sampling/sampling_logp_difference/mean": 0.01802951470017433, "step": 54, "step_time": 25.670479513239115 }, { "clip_ratio/high_max": 0.000749661133158952, "clip_ratio/high_mean": 0.000749661133158952, "clip_ratio/low_mean": 0.00012117865699110553, "clip_ratio/low_min": 0.00012117865699110553, "clip_ratio/region_mean": 0.0008708397857844829, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 2039.0, "completions/mean_length": 1065.1199951171875, "completions/mean_terminated_length": 931.0909423828125, "completions/min_length": 480.0, "completions/min_terminated_length": 480.0, "entropy": 0.22501038908958435, "epoch": 0.14945652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.021781066432595253, "learning_rate": 5e-05, "loss": 0.07857176661491394, "num_tokens": 3950277.0, "reward": 0.3199218809604645, "reward_std": 0.5881868600845337, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.5200781226158142, "rewards/length_penalty/std": 0.28003421425819397, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9916319251060486, "sampling/importance_sampling_ratio/min": 0.04320801421999931, "sampling/sampling_logp_difference/max": 3.1417293548583984, "sampling/sampling_logp_difference/mean": 0.021369094029068947, "step": 55, "step_time": 24.98296606983058 }, { "clip_ratio/high_max": 0.0002201356299337931, "clip_ratio/high_mean": 0.0002201356299337931, "clip_ratio/low_mean": 0.0001502036669990048, "clip_ratio/low_min": 0.0001502036669990048, "clip_ratio/region_mean": 0.000370339305663947, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1660.0, "completions/mean_length": 928.5, "completions/mean_terminated_length": 648.625, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.2072504162788391, "epoch": 0.15217391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.013964945450425148, "learning_rate": 5e-05, "loss": 0.05491746589541435, "num_tokens": 3999772.0, "reward": 0.14663085341453552, "reward_std": 0.6885973215103149, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.453369140625, "rewards/length_penalty/std": 0.30666640400886536, "sampling/importance_sampling_ratio/max": 2.906921625137329, "sampling/importance_sampling_ratio/mean": 0.9920700788497925, "sampling/importance_sampling_ratio/min": 0.03219592198729515, "sampling/sampling_logp_difference/max": 3.435915470123291, "sampling/sampling_logp_difference/mean": 0.02083013206720352, "step": 56, "step_time": 24.479757665889338 }, { "clip_ratio/high_max": 0.001786479854490608, "clip_ratio/high_mean": 0.001786479854490608, "clip_ratio/low_mean": 2.5886617368087173e-05, "clip_ratio/low_min": 2.5886617368087173e-05, "clip_ratio/region_mean": 0.0018123664776794612, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1872.0, "completions/mean_length": 1050.699951171875, "completions/mean_terminated_length": 801.375, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.2278908148407936, "epoch": 0.15489130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.014048721641302109, "learning_rate": 5e-05, "loss": 0.037405215203762054, "num_tokens": 4055547.0, "reward": 0.2869628965854645, "reward_std": 0.6820490956306458, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5130370855331421, "rewards/length_penalty/std": 0.32119351625442505, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.991212785243988, "sampling/importance_sampling_ratio/min": 0.018531763926148415, "sampling/sampling_logp_difference/max": 3.988269090652466, "sampling/sampling_logp_difference/mean": 0.021337997168302536, "step": 57, "step_time": 24.977965974714607 }, { "clip_ratio/high_max": 0.0005700029694708064, "clip_ratio/high_mean": 0.0005700029694708064, "clip_ratio/low_mean": 0.00015229834825731815, "clip_ratio/low_min": 0.00015229834825731815, "clip_ratio/region_mean": 0.0007223013206385076, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1868.0, "completions/mean_length": 1198.6199951171875, "completions/mean_terminated_length": 900.189208984375, "completions/min_length": 466.0, "completions/min_terminated_length": 466.0, "entropy": 0.255254665017128, "epoch": 0.15760869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.01905212551355362, "learning_rate": 5e-05, "loss": 0.09029528498649597, "num_tokens": 4119288.0, "reward": -0.0052636717446148396, "reward_std": 0.7467082142829895, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.5852636694908142, "rewards/length_penalty/std": 0.3044397234916687, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9905498623847961, "sampling/importance_sampling_ratio/min": 0.05557810142636299, "sampling/sampling_logp_difference/max": 2.8899660110473633, "sampling/sampling_logp_difference/mean": 0.023080680519342422, "step": 58, "step_time": 26.597216782160103 }, { "clip_ratio/high_max": 0.0008118896570522338, "clip_ratio/high_mean": 0.0008118896570522338, "clip_ratio/low_mean": 0.00015391591878142208, "clip_ratio/low_min": 0.00015391591878142208, "clip_ratio/region_mean": 0.0009658055671025068, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 677.4199829101562, "completions/mean_terminated_length": 649.448974609375, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.14910456240177156, "epoch": 0.16032608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.016270486637949944, "learning_rate": 5e-05, "loss": 0.04443974792957306, "num_tokens": 4155869.0, "reward": 0.5692285299301147, "reward_std": 0.390527606010437, "rewards/correctness/mean": 0.8999999761581421, "rewards/correctness/std": 0.30304574966430664, "rewards/length_penalty/mean": -0.33077147603034973, "rewards/length_penalty/std": 0.146135151386261, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9942877888679504, "sampling/importance_sampling_ratio/min": 0.08243148028850555, "sampling/sampling_logp_difference/max": 2.4957878589630127, "sampling/sampling_logp_difference/mean": 0.019903726875782013, "step": 59, "step_time": 22.8233163098339 }, { "clip_ratio/high_max": 0.0006108953821239993, "clip_ratio/high_mean": 0.0006108953821239993, "clip_ratio/low_mean": 8.367137343157083e-05, "clip_ratio/low_min": 8.367137343157083e-05, "clip_ratio/region_mean": 0.0006945667759282514, "completions/clipped_ratio": 0.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 740.719970703125, "completions/mean_terminated_length": 740.719970703125, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.1358457773923874, "epoch": 0.16304347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.01377896312624216, "learning_rate": 5e-05, "loss": 0.04055830091238022, "num_tokens": 4195715.0, "reward": 0.018320312723517418, "reward_std": 0.4930758476257324, "rewards/correctness/mean": 0.3799999952316284, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.36167967319488525, "rewards/length_penalty/std": 0.10894857347011566, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9946678876876831, "sampling/importance_sampling_ratio/min": 0.06510572135448456, "sampling/sampling_logp_difference/max": 2.7317428588867188, "sampling/sampling_logp_difference/mean": 0.01834091730415821, "step": 60, "step_time": 14.378700571134686 }, { "clip_ratio/high_max": 0.0007366885984083638, "clip_ratio/high_mean": 0.0007366885984083638, "clip_ratio/low_mean": 0.00010070936405099928, "clip_ratio/low_min": 0.00010070936405099928, "clip_ratio/region_mean": 0.0008373979799216613, "completions/clipped_ratio": 0.0, "completions/max_length": 896.0, "completions/max_terminated_length": 896.0, "completions/mean_length": 608.1199951171875, "completions/mean_terminated_length": 608.1199951171875, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "entropy": 0.1377216622233391, "epoch": 0.16576086956521738, "frac_reward_zero_std": 0.0, "grad_norm": 0.013815459795296192, "learning_rate": 5e-05, "loss": 0.033973757177591324, "num_tokens": 4228531.0, "reward": 0.7030664086341858, "reward_std": 0.04950352758169174, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.2969335913658142, "rewards/length_penalty/std": 0.049503523856401443, "sampling/importance_sampling_ratio/max": 2.5220999717712402, "sampling/importance_sampling_ratio/mean": 0.9949359893798828, "sampling/importance_sampling_ratio/min": 0.05055711790919304, "sampling/sampling_logp_difference/max": 2.984651565551758, "sampling/sampling_logp_difference/mean": 0.019737115129828453, "step": 61, "step_time": 11.692759637953714 }, { "clip_ratio/high_max": 0.0006575177831109613, "clip_ratio/high_mean": 0.0006575177831109613, "clip_ratio/low_mean": 6.264255061978474e-05, "clip_ratio/low_min": 6.264255061978474e-05, "clip_ratio/region_mean": 0.0007201603322755546, "completions/clipped_ratio": 0.0, "completions/max_length": 1995.0, "completions/max_terminated_length": 1995.0, "completions/mean_length": 890.6199951171875, "completions/mean_terminated_length": 890.6199951171875, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.14189326465129853, "epoch": 0.16847826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.015275136567652225, "learning_rate": 5e-05, "loss": 0.03717654570937157, "num_tokens": 4276682.0, "reward": 0.4051269292831421, "reward_std": 0.45795977115631104, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.4348730444908142, "rewards/length_penalty/std": 0.19076396524906158, "sampling/importance_sampling_ratio/max": 2.618985414505005, "sampling/importance_sampling_ratio/mean": 0.9940707683563232, "sampling/importance_sampling_ratio/min": 0.038013607263565063, "sampling/sampling_logp_difference/max": 3.2698111534118652, "sampling/sampling_logp_difference/mean": 0.016751455143094063, "step": 62, "step_time": 23.50743620051071 }, { "clip_ratio/high_max": 0.0008010541961994023, "clip_ratio/high_mean": 0.0008010541961994023, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008010541961994023, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1916.0, "completions/mean_length": 706.97998046875, "completions/mean_terminated_length": 679.6122436523438, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.1526895821094513, "epoch": 0.17119565217391305, "frac_reward_zero_std": 0.0, "grad_norm": 0.01731613650918007, "learning_rate": 5e-05, "loss": 0.04923119395971298, "num_tokens": 4315281.0, "reward": 0.45479491353034973, "reward_std": 0.5850844383239746, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.34520506858825684, "rewards/length_penalty/std": 0.20253480970859528, "sampling/importance_sampling_ratio/max": 2.8465676307678223, "sampling/importance_sampling_ratio/mean": 0.9949560165405273, "sampling/importance_sampling_ratio/min": 0.27247118949890137, "sampling/sampling_logp_difference/max": 1.300222396850586, "sampling/sampling_logp_difference/mean": 0.019272862002253532, "step": 63, "step_time": 23.484382715076208 }, { "clip_ratio/high_max": 0.0003500981023535132, "clip_ratio/high_mean": 0.0003500981023535132, "clip_ratio/low_mean": 5.712283600587398e-05, "clip_ratio/low_min": 5.712283600587398e-05, "clip_ratio/region_mean": 0.00040722094126977026, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.0, "completions/max_terminated_length": 1117.0, "completions/mean_length": 700.2999877929688, "completions/mean_terminated_length": 700.2999877929688, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.13254741728305816, "epoch": 0.17391304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.01813482865691185, "learning_rate": 5e-05, "loss": 0.023750657215714455, "num_tokens": 4353816.0, "reward": 0.6180566549301147, "reward_std": 0.22116827964782715, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.3419433534145355, "rewards/length_penalty/std": 0.09352646768093109, "sampling/importance_sampling_ratio/max": 2.9889914989471436, "sampling/importance_sampling_ratio/mean": 0.9951876401901245, "sampling/importance_sampling_ratio/min": 0.07279559969902039, "sampling/sampling_logp_difference/max": 2.6200997829437256, "sampling/sampling_logp_difference/mean": 0.018573181703686714, "step": 64, "step_time": 13.950854809721932 }, { "clip_ratio/high_max": 0.0010051134566310793, "clip_ratio/high_mean": 0.0010051134566310793, "clip_ratio/low_mean": 7.008167740423233e-05, "clip_ratio/low_min": 7.008167740423233e-05, "clip_ratio/region_mean": 0.0010751951369456947, "completions/clipped_ratio": 0.0, "completions/max_length": 1941.0, "completions/max_terminated_length": 1941.0, "completions/mean_length": 760.8599853515625, "completions/mean_terminated_length": 760.8599853515625, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.15310844779014587, "epoch": 0.1766304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.01888042315840721, "learning_rate": 5e-05, "loss": 0.04399148002266884, "num_tokens": 4395569.0, "reward": 0.5684863328933716, "reward_std": 0.33382996916770935, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979365825653, "rewards/length_penalty/mean": -0.3715136647224426, "rewards/length_penalty/std": 0.16755269467830658, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9941285848617554, "sampling/importance_sampling_ratio/min": 0.07204371690750122, "sampling/sampling_logp_difference/max": 2.6304821968078613, "sampling/sampling_logp_difference/mean": 0.01957661285996437, "step": 65, "step_time": 23.068237875122577 }, { "clip_ratio/high_max": 0.0005175740210688673, "clip_ratio/high_mean": 0.0005175740210688673, "clip_ratio/low_mean": 0.00023222646414069458, "clip_ratio/low_min": 0.00023222646414069458, "clip_ratio/region_mean": 0.0007498004793887958, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 1759.0, "completions/mean_length": 757.97998046875, "completions/mean_terminated_length": 547.9767456054688, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.25827352702617645, "epoch": 0.1793478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.016815872862935066, "learning_rate": 5e-05, "loss": 0.036842163652181625, "num_tokens": 4436518.0, "reward": 0.389892578125, "reward_std": 0.6972166895866394, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.37010741233825684, "rewards/length_penalty/std": 0.28814491629600525, "sampling/importance_sampling_ratio/max": 2.3030846118927, "sampling/importance_sampling_ratio/mean": 0.9900518655776978, "sampling/importance_sampling_ratio/min": 0.16431906819343567, "sampling/sampling_logp_difference/max": 1.8059451580047607, "sampling/sampling_logp_difference/mean": 0.026268478482961655, "step": 66, "step_time": 23.87718580919318 }, { "clip_ratio/high_max": 0.00029754412826150657, "clip_ratio/high_mean": 0.00029754412826150657, "clip_ratio/low_mean": 3.082613984588534e-05, "clip_ratio/low_min": 3.082613984588534e-05, "clip_ratio/region_mean": 0.00032837026519700886, "completions/clipped_ratio": 0.0, "completions/max_length": 1330.0, "completions/max_terminated_length": 1330.0, "completions/mean_length": 671.2000122070312, "completions/mean_terminated_length": 671.2000122070312, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 0.10958393663167953, "epoch": 0.18206521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.014129486866295338, "learning_rate": 5e-05, "loss": 0.03558261692523956, "num_tokens": 4473028.0, "reward": 0.27226561307907104, "reward_std": 0.5440670847892761, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.3277343809604645, "rewards/length_penalty/std": 0.13625548779964447, "sampling/importance_sampling_ratio/max": 2.361050605773926, "sampling/importance_sampling_ratio/mean": 0.996082603931427, "sampling/importance_sampling_ratio/min": 0.0901251807808876, "sampling/sampling_logp_difference/max": 2.406555652618408, "sampling/sampling_logp_difference/mean": 0.016017161309719086, "step": 67, "step_time": 15.848499092971906 }, { "clip_ratio/high_max": 0.0004814961168449372, "clip_ratio/high_mean": 0.0004814961168449372, "clip_ratio/low_mean": 0.00011306432425044477, "clip_ratio/low_min": 0.00011306432425044477, "clip_ratio/region_mean": 0.000594560446916148, "completions/clipped_ratio": 0.0, "completions/max_length": 1327.0, "completions/max_terminated_length": 1327.0, "completions/mean_length": 709.97998046875, "completions/mean_terminated_length": 709.97998046875, "completions/min_length": 377.0, "completions/min_terminated_length": 377.0, "entropy": 0.12251516133546829, "epoch": 0.18478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.017992310225963593, "learning_rate": 5e-05, "loss": 0.034500207751989365, "num_tokens": 4512287.0, "reward": 0.43333005905151367, "reward_std": 0.5297770500183105, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.34666991233825684, "rewards/length_penalty/std": 0.12326952069997787, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9955129027366638, "sampling/importance_sampling_ratio/min": 0.2864378094673157, "sampling/sampling_logp_difference/max": 1.2502338886260986, "sampling/sampling_logp_difference/mean": 0.01752791553735733, "step": 68, "step_time": 16.056356735294685 }, { "clip_ratio/high_max": 0.000786409858847037, "clip_ratio/high_mean": 0.000786409858847037, "clip_ratio/low_mean": 0.00017479720409028233, "clip_ratio/low_min": 0.00017479720409028233, "clip_ratio/region_mean": 0.0009612070745788515, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1685.0, "completions/mean_length": 695.0, "completions/mean_terminated_length": 544.6666870117188, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.2089858502149582, "epoch": 0.1875, "frac_reward_zero_std": 0.0, "grad_norm": 0.01573641039431095, "learning_rate": 5e-05, "loss": 0.06109810620546341, "num_tokens": 4549417.0, "reward": 0.2606445252895355, "reward_std": 0.6902934312820435, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.33935546875, "rewards/length_penalty/std": 0.27389469742774963, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9919227361679077, "sampling/importance_sampling_ratio/min": 0.04992447420954704, "sampling/sampling_logp_difference/max": 2.997243881225586, "sampling/sampling_logp_difference/mean": 0.022705664858222008, "step": 69, "step_time": 24.100224602036178 }, { "clip_ratio/high_max": 0.0003861734687234275, "clip_ratio/high_mean": 0.0003861734687234275, "clip_ratio/low_mean": 8.133393421303481e-05, "clip_ratio/low_min": 8.133393421303481e-05, "clip_ratio/region_mean": 0.00046750739857088776, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1819.0, "completions/mean_length": 974.5999755859375, "completions/mean_terminated_length": 706.25, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.12589940428733826, "epoch": 0.19021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.014950593002140522, "learning_rate": 5e-05, "loss": 0.03755060210824013, "num_tokens": 4601657.0, "reward": -0.27587890625, "reward_std": 0.6061694622039795, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.47587889432907104, "rewards/length_penalty/std": 0.3194302022457123, "sampling/importance_sampling_ratio/max": 2.5902318954467773, "sampling/importance_sampling_ratio/mean": 0.9951319098472595, "sampling/importance_sampling_ratio/min": 0.07221422344446182, "sampling/sampling_logp_difference/max": 2.6281182765960693, "sampling/sampling_logp_difference/mean": 0.015436016023159027, "step": 70, "step_time": 24.753103533759713 }, { "clip_ratio/high_max": 0.0009733674465678632, "clip_ratio/high_mean": 0.0009733674465678632, "clip_ratio/low_mean": 0.00012668214039877058, "clip_ratio/low_min": 0.00012668214039877058, "clip_ratio/region_mean": 0.001100049610249698, "completions/clipped_ratio": 0.0, "completions/max_length": 1306.0, "completions/max_terminated_length": 1306.0, "completions/mean_length": 481.8999938964844, "completions/mean_terminated_length": 481.8999938964844, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.18031663894653321, "epoch": 0.19293478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.017065592110157013, "learning_rate": 5e-05, "loss": 0.02791469730436802, "num_tokens": 4628112.0, "reward": 0.26469725370407104, "reward_std": 0.5313357710838318, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.23530273139476776, "rewards/length_penalty/std": 0.08668895810842514, "sampling/importance_sampling_ratio/max": 2.9714601039886475, "sampling/importance_sampling_ratio/mean": 0.9928731918334961, "sampling/importance_sampling_ratio/min": 0.06918845325708389, "sampling/sampling_logp_difference/max": 2.6709213256835938, "sampling/sampling_logp_difference/mean": 0.026175597682595253, "step": 71, "step_time": 14.497095804661512 }, { "clip_ratio/high_max": 0.000628325465368107, "clip_ratio/high_mean": 0.000628325465368107, "clip_ratio/low_mean": 4.015257873106748e-05, "clip_ratio/low_min": 4.015257873106748e-05, "clip_ratio/region_mean": 0.0006684780470095575, "completions/clipped_ratio": 0.0, "completions/max_length": 1143.0, "completions/max_terminated_length": 1143.0, "completions/mean_length": 483.8999938964844, "completions/mean_terminated_length": 483.8999938964844, "completions/min_length": 178.0, "completions/min_terminated_length": 178.0, "entropy": 0.11079794615507126, "epoch": 0.1956521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.01392416376620531, "learning_rate": 5e-05, "loss": 0.02589317597448826, "num_tokens": 4655977.0, "reward": 0.5037207007408142, "reward_std": 0.5148534774780273, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.23627929389476776, "rewards/length_penalty/std": 0.10259860754013062, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9951469898223877, "sampling/importance_sampling_ratio/min": 0.2387021780014038, "sampling/sampling_logp_difference/max": 1.432538628578186, "sampling/sampling_logp_difference/mean": 0.01793760433793068, "step": 72, "step_time": 13.306627204176039 }, { "clip_ratio/high_max": 0.0005853123526321724, "clip_ratio/high_mean": 0.0005853123526321724, "clip_ratio/low_mean": 0.0001358868816168979, "clip_ratio/low_min": 0.0001358868816168979, "clip_ratio/region_mean": 0.0007211992342490703, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 2014.0, "completions/mean_length": 723.5999755859375, "completions/mean_terminated_length": 608.434814453125, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.17599567174911498, "epoch": 0.1983695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.018252408131957054, "learning_rate": 5e-05, "loss": 0.04385628551244736, "num_tokens": 4694567.0, "reward": 0.22667968273162842, "reward_std": 0.6738052368164062, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.35332030057907104, "rewards/length_penalty/std": 0.26818665862083435, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9931528568267822, "sampling/importance_sampling_ratio/min": 0.20481857657432556, "sampling/sampling_logp_difference/max": 1.5856306552886963, "sampling/sampling_logp_difference/mean": 0.020317068323493004, "step": 73, "step_time": 24.120045095914975 }, { "clip_ratio/high_max": 0.0005197267164476216, "clip_ratio/high_mean": 0.0005197267164476216, "clip_ratio/low_mean": 7.821666076779366e-05, "clip_ratio/low_min": 7.821666076779366e-05, "clip_ratio/region_mean": 0.0005979433772154152, "completions/clipped_ratio": 0.0, "completions/max_length": 916.0, "completions/max_terminated_length": 916.0, "completions/mean_length": 504.55999755859375, "completions/mean_terminated_length": 504.55999755859375, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.14893011450767518, "epoch": 0.20108695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.015567939728498459, "learning_rate": 5e-05, "loss": 0.03300417959690094, "num_tokens": 4723125.0, "reward": 0.15363280475139618, "reward_std": 0.5294827818870544, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.2463671863079071, "rewards/length_penalty/std": 0.09036648273468018, "sampling/importance_sampling_ratio/max": 2.6094131469726562, "sampling/importance_sampling_ratio/mean": 0.9946196675300598, "sampling/importance_sampling_ratio/min": 0.20717699825763702, "sampling/sampling_logp_difference/max": 1.5741817951202393, "sampling/sampling_logp_difference/mean": 0.02010997198522091, "step": 74, "step_time": 11.145676413783804 }, { "clip_ratio/high_max": 0.0009385790792293847, "clip_ratio/high_mean": 0.0009385790792293847, "clip_ratio/low_mean": 0.00018966651987284422, "clip_ratio/low_min": 0.00018966651987284422, "clip_ratio/region_mean": 0.001128245610743761, "completions/clipped_ratio": 0.0, "completions/max_length": 1005.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 429.67999267578125, "completions/mean_terminated_length": 429.67999267578125, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "entropy": 0.1344812273979187, "epoch": 0.20380434782608695, "frac_reward_zero_std": 0.0, "grad_norm": 0.01469445414841175, "learning_rate": 5e-05, "loss": 0.01116997841745615, "num_tokens": 4746939.0, "reward": 0.6301953196525574, "reward_std": 0.38830801844596863, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.2098046839237213, "rewards/length_penalty/std": 0.12929898500442505, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9950018525123596, "sampling/importance_sampling_ratio/min": 0.17992667853832245, "sampling/sampling_logp_difference/max": 1.7152059078216553, "sampling/sampling_logp_difference/mean": 0.022754253819584846, "step": 75, "step_time": 11.691951024346054 }, { "clip_ratio/high_max": 0.00032894672185648235, "clip_ratio/high_mean": 0.00032894672185648235, "clip_ratio/low_mean": 0.0002415040013147518, "clip_ratio/low_min": 0.0002415040013147518, "clip_ratio/region_mean": 0.0005704507231712342, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 2009.0, "completions/mean_length": 591.8200073242188, "completions/mean_terminated_length": 430.0222473144531, "completions/min_length": 148.0, "completions/min_terminated_length": 148.0, "entropy": 0.26585462391376496, "epoch": 0.20652173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.014526180922985077, "learning_rate": 5e-05, "loss": 0.04992569983005524, "num_tokens": 4779870.0, "reward": 0.4910253882408142, "reward_std": 0.6834759712219238, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.2889746129512787, "rewards/length_penalty/std": 0.2923096716403961, "sampling/importance_sampling_ratio/max": 2.685748338699341, "sampling/importance_sampling_ratio/mean": 0.9892884492874146, "sampling/importance_sampling_ratio/min": 0.30007389187812805, "sampling/sampling_logp_difference/max": 1.2037265300750732, "sampling/sampling_logp_difference/mean": 0.0263529010117054, "step": 76, "step_time": 23.11679401784204 }, { "clip_ratio/high_max": 0.0008211379870772361, "clip_ratio/high_mean": 0.0008211379870772361, "clip_ratio/low_mean": 8.669763628859073e-05, "clip_ratio/low_min": 8.669763628859073e-05, "clip_ratio/region_mean": 0.00090783562627621, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1921.0, "completions/mean_length": 535.6799926757812, "completions/mean_terminated_length": 472.66668701171875, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.17671192586421966, "epoch": 0.20923913043478262, "frac_reward_zero_std": 0.0, "grad_norm": 0.01664687879383564, "learning_rate": 5e-05, "loss": 0.04311929643154144, "num_tokens": 4809214.0, "reward": 0.5584374666213989, "reward_std": 0.5280290246009827, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.2615624964237213, "rewards/length_penalty/std": 0.23831947147846222, "sampling/importance_sampling_ratio/max": 2.370129108428955, "sampling/importance_sampling_ratio/mean": 0.9928213953971863, "sampling/importance_sampling_ratio/min": 0.16383084654808044, "sampling/sampling_logp_difference/max": 1.8089207410812378, "sampling/sampling_logp_difference/mean": 0.019636210054159164, "step": 77, "step_time": 22.68614157452248 }, { "clip_ratio/high_max": 0.0005915068177273497, "clip_ratio/high_mean": 0.0005915068177273497, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0005915068177273497, "completions/clipped_ratio": 0.0, "completions/max_length": 1322.0, "completions/max_terminated_length": 1322.0, "completions/mean_length": 444.0199890136719, "completions/mean_terminated_length": 444.0199890136719, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.09074169397354126, "epoch": 0.21195652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.013410658575594425, "learning_rate": 5e-05, "loss": 0.04930553585290909, "num_tokens": 4833715.0, "reward": 0.38319334387779236, "reward_std": 0.5701014995574951, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.21680663526058197, "rewards/length_penalty/std": 0.1205679401755333, "sampling/importance_sampling_ratio/max": 2.962005138397217, "sampling/importance_sampling_ratio/mean": 0.9963091015815735, "sampling/importance_sampling_ratio/min": 0.15177083015441895, "sampling/sampling_logp_difference/max": 1.8853836059570312, "sampling/sampling_logp_difference/mean": 0.014590539038181305, "step": 78, "step_time": 15.235982230165973 }, { "clip_ratio/high_max": 0.0005698791879694909, "clip_ratio/high_mean": 0.0005698791879694909, "clip_ratio/low_mean": 4.1485170368105176e-05, "clip_ratio/low_min": 4.1485170368105176e-05, "clip_ratio/region_mean": 0.0006113643466960639, "completions/clipped_ratio": 0.0, "completions/max_length": 700.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 444.0199890136719, "completions/mean_terminated_length": 444.0199890136719, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.10524996668100357, "epoch": 0.21467391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.013324566185474396, "learning_rate": 5e-05, "loss": 0.010624676942825317, "num_tokens": 4859376.0, "reward": 0.5031933188438416, "reward_std": 0.43716707825660706, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.21680663526058197, "rewards/length_penalty/std": 0.06869058310985565, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9948770403862, "sampling/importance_sampling_ratio/min": 0.3475484549999237, "sampling/sampling_logp_difference/max": 1.1924397945404053, "sampling/sampling_logp_difference/mean": 0.019135504961013794, "step": 79, "step_time": 8.87365493294783 }, { "clip_ratio/high_max": 0.00045147799828555436, "clip_ratio/high_mean": 0.00045147799828555436, "clip_ratio/low_mean": 0.0001845527091063559, "clip_ratio/low_min": 0.0001845527091063559, "clip_ratio/region_mean": 0.0006360307103022933, "completions/clipped_ratio": 0.0, "completions/max_length": 665.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 409.3800048828125, "completions/mean_terminated_length": 409.3800048828125, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 0.11959978342056274, "epoch": 0.21739130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.013123178854584694, "learning_rate": 5e-05, "loss": 0.024036118760704994, "num_tokens": 4883005.0, "reward": 0.2001074105501175, "reward_std": 0.5326245427131653, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.1998925805091858, "rewards/length_penalty/std": 0.05713106691837311, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9956418871879578, "sampling/importance_sampling_ratio/min": 0.3026818335056305, "sampling/sampling_logp_difference/max": 1.1951258182525635, "sampling/sampling_logp_difference/mean": 0.02054966241121292, "step": 80, "step_time": 8.360282597364858 }, { "clip_ratio/high_max": 0.000704864104045555, "clip_ratio/high_mean": 0.000704864104045555, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.000704864104045555, "completions/clipped_ratio": 0.0, "completions/max_length": 818.0, "completions/max_terminated_length": 818.0, "completions/mean_length": 417.5799865722656, "completions/mean_terminated_length": 417.5799865722656, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.1103743776679039, "epoch": 0.22010869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.01447275374084711, "learning_rate": 5e-05, "loss": 0.021041925996541977, "num_tokens": 4908344.0, "reward": 0.556103527545929, "reward_std": 0.4081662595272064, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.20389647781848907, "rewards/length_penalty/std": 0.07568074017763138, "sampling/importance_sampling_ratio/max": 2.083041191101074, "sampling/importance_sampling_ratio/mean": 0.9952544569969177, "sampling/importance_sampling_ratio/min": 0.2227083295583725, "sampling/sampling_logp_difference/max": 1.501892328262329, "sampling/sampling_logp_difference/mean": 0.01730303466320038, "step": 81, "step_time": 10.241758018033579 }, { "clip_ratio/high_max": 0.0006575023173354567, "clip_ratio/high_mean": 0.0006575023173354567, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0006575023173354567, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 294.55999755859375, "completions/mean_terminated_length": 294.55999755859375, "completions/min_length": 119.0, "completions/min_terminated_length": 119.0, "entropy": 0.10942234545946121, "epoch": 0.22282608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.012251164764165878, "learning_rate": 5e-05, "loss": 0.009689167141914368, "num_tokens": 4925922.0, "reward": 0.5361718535423279, "reward_std": 0.5191479921340942, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.4712120592594147, "rewards/length_penalty/mean": -0.1438281238079071, "rewards/length_penalty/std": 0.07002057135105133, "sampling/importance_sampling_ratio/max": 2.1058547496795654, "sampling/importance_sampling_ratio/mean": 0.997086763381958, "sampling/importance_sampling_ratio/min": 0.2412642240524292, "sampling/sampling_logp_difference/max": 1.4218626022338867, "sampling/sampling_logp_difference/mean": 0.018847452476620674, "step": 82, "step_time": 7.392252362798899 }, { "clip_ratio/high_max": 0.00035982736153528094, "clip_ratio/high_mean": 0.00035982736153528094, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00035982736153528094, "completions/clipped_ratio": 0.0, "completions/max_length": 491.0, "completions/max_terminated_length": 491.0, "completions/mean_length": 237.3199920654297, "completions/mean_terminated_length": 237.3199920654297, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "entropy": 0.1137263223528862, "epoch": 0.22554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.012257511727511883, "learning_rate": 5e-05, "loss": 0.013772832229733467, "num_tokens": 4940218.0, "reward": 0.4641210734844208, "reward_std": 0.5065000653266907, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.11587890982627869, "rewards/length_penalty/std": 0.04924225062131882, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9956432580947876, "sampling/importance_sampling_ratio/min": 0.24330933392047882, "sampling/sampling_logp_difference/max": 1.413421630859375, "sampling/sampling_logp_difference/mean": 0.018884913995862007, "step": 83, "step_time": 6.00911526218988 }, { "clip_ratio/high_max": 0.0010932499542832374, "clip_ratio/high_mean": 0.0010932499542832374, "clip_ratio/low_mean": 5.839415825903416e-05, "clip_ratio/low_min": 5.839415825903416e-05, "clip_ratio/region_mean": 0.0011516441009007394, "completions/clipped_ratio": 0.0, "completions/max_length": 1617.0, "completions/max_terminated_length": 1617.0, "completions/mean_length": 375.1600036621094, "completions/mean_terminated_length": 375.1600036621094, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "entropy": 0.1211256891489029, "epoch": 0.22826086956521738, "frac_reward_zero_std": 0.0, "grad_norm": 0.019089948385953903, "learning_rate": 5e-05, "loss": 0.03538642078638077, "num_tokens": 4961956.0, "reward": 0.5568163990974426, "reward_std": 0.4542848467826843, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.18318359553813934, "rewards/length_penalty/std": 0.18591263890266418, "sampling/importance_sampling_ratio/max": 2.340315580368042, "sampling/importance_sampling_ratio/mean": 0.995381236076355, "sampling/importance_sampling_ratio/min": 0.24651946127414703, "sampling/sampling_logp_difference/max": 1.4003143310546875, "sampling/sampling_logp_difference/mean": 0.018075764179229736, "step": 84, "step_time": 18.067105296999216 }, { "clip_ratio/high_max": 0.0010971381212584673, "clip_ratio/high_mean": 0.0010971381212584673, "clip_ratio/low_mean": 9.907572530210018e-05, "clip_ratio/low_min": 9.907572530210018e-05, "clip_ratio/region_mean": 0.0011962138232775033, "completions/clipped_ratio": 0.0, "completions/max_length": 1556.0, "completions/max_terminated_length": 1556.0, "completions/mean_length": 362.79998779296875, "completions/mean_terminated_length": 362.79998779296875, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 0.2110247790813446, "epoch": 0.23097826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.021158233284950256, "learning_rate": 5e-05, "loss": 0.044072747230529785, "num_tokens": 4982116.0, "reward": 0.3828515410423279, "reward_std": 0.5390759110450745, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.17714843153953552, "rewards/length_penalty/std": 0.12832064926624298, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9906708598136902, "sampling/importance_sampling_ratio/min": 0.171915665268898, "sampling/sampling_logp_difference/max": 1.7607512474060059, "sampling/sampling_logp_difference/mean": 0.028823882341384888, "step": 85, "step_time": 16.47355701122433 }, { "clip_ratio/high_max": 0.0005582125799264759, "clip_ratio/high_mean": 0.0005582125799264759, "clip_ratio/low_mean": 0.00011848341673612595, "clip_ratio/low_min": 0.00011848341673612595, "clip_ratio/region_mean": 0.0006766959966626018, "completions/clipped_ratio": 0.0, "completions/max_length": 332.0, "completions/max_terminated_length": 332.0, "completions/mean_length": 183.47999572753906, "completions/mean_terminated_length": 183.47999572753906, "completions/min_length": 57.0, "completions/min_terminated_length": 57.0, "entropy": 0.14162901043891907, "epoch": 0.23369565217391305, "frac_reward_zero_std": 0.0, "grad_norm": 0.01095115952193737, "learning_rate": 5e-05, "loss": 0.007650045212358236, "num_tokens": 4993390.0, "reward": 0.8504101634025574, "reward_std": 0.24047701060771942, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979514837265, "rewards/length_penalty/mean": -0.08958984166383743, "rewards/length_penalty/std": 0.0338219590485096, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9944058656692505, "sampling/importance_sampling_ratio/min": 0.1472282111644745, "sampling/sampling_logp_difference/max": 1.915771484375, "sampling/sampling_logp_difference/mean": 0.02622547745704651, "step": 86, "step_time": 4.271973532857373 }, { "clip_ratio/high_max": 0.0005829900386743247, "clip_ratio/high_mean": 0.0005829900386743247, "clip_ratio/low_mean": 0.0003810461843386292, "clip_ratio/low_min": 0.0003810461843386292, "clip_ratio/region_mean": 0.0009640362113714218, "completions/clipped_ratio": 0.0, "completions/max_length": 630.0, "completions/max_terminated_length": 630.0, "completions/mean_length": 281.739990234375, "completions/mean_terminated_length": 281.739990234375, "completions/min_length": 74.0, "completions/min_terminated_length": 74.0, "entropy": 0.10393131375312806, "epoch": 0.23641304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.013417479582130909, "learning_rate": 5e-05, "loss": 0.014699541963636875, "num_tokens": 5009687.0, "reward": 0.422431617975235, "reward_std": 0.5398931503295898, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.13756835460662842, "rewards/length_penalty/std": 0.07445195317268372, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9958858489990234, "sampling/importance_sampling_ratio/min": 0.13931959867477417, "sampling/sampling_logp_difference/max": 1.970984697341919, "sampling/sampling_logp_difference/mean": 0.018261417746543884, "step": 87, "step_time": 7.5571192188654095 }, { "clip_ratio/high_max": 0.0015571706928312779, "clip_ratio/high_mean": 0.0015571706928312779, "clip_ratio/low_mean": 0.0001285347039811313, "clip_ratio/low_min": 0.0001285347039811313, "clip_ratio/region_mean": 0.0016857054084539413, "completions/clipped_ratio": 0.0, "completions/max_length": 400.0, "completions/max_terminated_length": 400.0, "completions/mean_length": 182.1599884033203, "completions/mean_terminated_length": 182.1599884033203, "completions/min_length": 41.0, "completions/min_terminated_length": 41.0, "entropy": 0.15183552503585815, "epoch": 0.2391304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.011707360856235027, "learning_rate": 5e-05, "loss": 0.0038167578168213367, "num_tokens": 5021685.0, "reward": 0.6910547018051147, "reward_std": 0.4340381622314453, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.08894531428813934, "rewards/length_penalty/std": 0.052339497953653336, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9942026138305664, "sampling/importance_sampling_ratio/min": 0.1707821637392044, "sampling/sampling_logp_difference/max": 1.7673664093017578, "sampling/sampling_logp_difference/mean": 0.02865365333855152, "step": 88, "step_time": 5.0701005139853805 }, { "clip_ratio/high_max": 0.0011892693408299237, "clip_ratio/high_mean": 0.0011892693408299237, "clip_ratio/low_mean": 0.000522052450105548, "clip_ratio/low_min": 0.000522052450105548, "clip_ratio/region_mean": 0.0017113217909354717, "completions/clipped_ratio": 0.0, "completions/max_length": 512.0, "completions/max_terminated_length": 512.0, "completions/mean_length": 213.0, "completions/mean_terminated_length": 213.0, "completions/min_length": 63.0, "completions/min_terminated_length": 63.0, "entropy": 0.15271458625793458, "epoch": 0.2418478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.010374904610216618, "learning_rate": 5e-05, "loss": 0.011677139438688755, "num_tokens": 5035795.0, "reward": 0.3359960913658142, "reward_std": 0.5273035168647766, "rewards/correctness/mean": 0.4399999976158142, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.10400390625, "rewards/length_penalty/std": 0.061812058091163635, "sampling/importance_sampling_ratio/max": 2.3226864337921143, "sampling/importance_sampling_ratio/mean": 0.993018627166748, "sampling/importance_sampling_ratio/min": 0.08702941983938217, "sampling/sampling_logp_difference/max": 2.4415090084075928, "sampling/sampling_logp_difference/mean": 0.02671627141535282, "step": 89, "step_time": 6.264251670800149 }, { "clip_ratio/high_max": 0.0012449576752260327, "clip_ratio/high_mean": 0.0012449576752260327, "clip_ratio/low_mean": 0.00039986594929359854, "clip_ratio/low_min": 0.00039986594929359854, "clip_ratio/region_mean": 0.001644823612878099, "completions/clipped_ratio": 0.0, "completions/max_length": 1107.0, "completions/max_terminated_length": 1107.0, "completions/mean_length": 258.67999267578125, "completions/mean_terminated_length": 258.67999267578125, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "entropy": 0.17646301686763763, "epoch": 0.24456521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.013736448250710964, "learning_rate": 5e-05, "loss": 0.030243348330259323, "num_tokens": 5051759.0, "reward": 0.53369140625, "reward_std": 0.510785698890686, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.47851815819740295, "rewards/length_penalty/mean": -0.1263085901737213, "rewards/length_penalty/std": 0.08943063020706177, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9928296804428101, "sampling/importance_sampling_ratio/min": 0.09859311580657959, "sampling/sampling_logp_difference/max": 2.31675386428833, "sampling/sampling_logp_difference/mean": 0.028476905077695847, "step": 90, "step_time": 11.927960007218644 }, { "clip_ratio/high_max": 0.0011836130055598915, "clip_ratio/high_mean": 0.0011836130055598915, "clip_ratio/low_mean": 0.0006012129248119891, "clip_ratio/low_min": 0.0006012129248119891, "clip_ratio/region_mean": 0.0017848259536549448, "completions/clipped_ratio": 0.0, "completions/max_length": 784.0, "completions/max_terminated_length": 784.0, "completions/mean_length": 168.05999755859375, "completions/mean_terminated_length": 168.05999755859375, "completions/min_length": 43.0, "completions/min_terminated_length": 43.0, "entropy": 0.1712634652853012, "epoch": 0.24728260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.011951963417232037, "learning_rate": 5e-05, "loss": 0.021161450073122978, "num_tokens": 5065332.0, "reward": 0.5579394698143005, "reward_std": 0.5229561924934387, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.0820605456829071, "rewards/length_penalty/std": 0.08735550940036774, "sampling/importance_sampling_ratio/max": 2.185065507888794, "sampling/importance_sampling_ratio/mean": 0.9940653443336487, "sampling/importance_sampling_ratio/min": 0.12851713597774506, "sampling/sampling_logp_difference/max": 2.0516929626464844, "sampling/sampling_logp_difference/mean": 0.02416444756090641, "step": 91, "step_time": 9.349531583720818 }, { "clip_ratio/high_max": 0.0010452893213368953, "clip_ratio/high_mean": 0.0010452893213368953, "clip_ratio/low_mean": 0.00012650220887735485, "clip_ratio/low_min": 0.00012650220887735485, "clip_ratio/region_mean": 0.00117179153021425, "completions/clipped_ratio": 0.0, "completions/max_length": 292.0, "completions/max_terminated_length": 292.0, "completions/mean_length": 136.17999267578125, "completions/mean_terminated_length": 136.17999267578125, "completions/min_length": 51.0, "completions/min_terminated_length": 51.0, "entropy": 0.15401647984981537, "epoch": 0.25, "frac_reward_zero_std": 0.0, "grad_norm": 0.015128343366086483, "learning_rate": 5e-05, "loss": 0.008885608986020088, "num_tokens": 5074531.0, "reward": 0.5135058760643005, "reward_std": 0.5054919719696045, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.06649413704872131, "rewards/length_penalty/std": 0.029116734862327576, "sampling/importance_sampling_ratio/max": 2.3821372985839844, "sampling/importance_sampling_ratio/mean": 0.9937571287155151, "sampling/importance_sampling_ratio/min": 0.09059903770685196, "sampling/sampling_logp_difference/max": 2.4013116359710693, "sampling/sampling_logp_difference/mean": 0.024070926010608673, "step": 92, "step_time": 4.327101199189201 }, { "clip_ratio/high_max": 0.0013822196982800961, "clip_ratio/high_mean": 0.0013822196982800961, "clip_ratio/low_mean": 0.0005461856024339795, "clip_ratio/low_min": 0.0005461856024339795, "clip_ratio/region_mean": 0.0019284053007140756, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1163.0, "completions/mean_length": 264.32000732421875, "completions/mean_terminated_length": 190.0, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "entropy": 0.2661470055580139, "epoch": 0.25271739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.020688654854893684, "learning_rate": 5e-05, "loss": 0.10969983786344528, "num_tokens": 5093367.0, "reward": 0.5709375143051147, "reward_std": 0.6100456118583679, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.1290625035762787, "rewards/length_penalty/std": 0.21535643935203552, "sampling/importance_sampling_ratio/max": 2.3409512042999268, "sampling/importance_sampling_ratio/mean": 0.990288496017456, "sampling/importance_sampling_ratio/min": 0.213004007935524, "sampling/sampling_logp_difference/max": 1.546444296836853, "sampling/sampling_logp_difference/mean": 0.02781936153769493, "step": 93, "step_time": 22.407282277243212 }, { "clip_ratio/high_max": 0.0015437419177033007, "clip_ratio/high_mean": 0.0015437419177033007, "clip_ratio/low_mean": 0.00012154360301792621, "clip_ratio/low_min": 0.00012154360301792621, "clip_ratio/region_mean": 0.0016652855207212268, "completions/clipped_ratio": 0.0, "completions/max_length": 1413.0, "completions/max_terminated_length": 1413.0, "completions/mean_length": 220.3199920654297, "completions/mean_terminated_length": 220.3199920654297, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 0.22524437308311462, "epoch": 0.2554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.020771974697709084, "learning_rate": 5e-05, "loss": 0.052876319736242294, "num_tokens": 5110413.0, "reward": 0.412421852350235, "reward_std": 0.5003185272216797, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.10757812857627869, "rewards/length_penalty/std": 0.1135673075914383, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9909684062004089, "sampling/importance_sampling_ratio/min": 0.32960575819015503, "sampling/sampling_logp_difference/max": 1.5696799755096436, "sampling/sampling_logp_difference/mean": 0.028162555769085884, "step": 94, "step_time": 15.704104893608019 }, { "clip_ratio/high_max": 0.001439261925406754, "clip_ratio/high_mean": 0.001439261925406754, "clip_ratio/low_mean": 0.00019417476141825318, "clip_ratio/low_min": 0.00019417476141825318, "clip_ratio/region_mean": 0.0016334366519004107, "completions/clipped_ratio": 0.0, "completions/max_length": 194.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 97.45999908447266, "completions/mean_terminated_length": 97.45999908447266, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "entropy": 0.25395846366882324, "epoch": 0.25815217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.011895176954567432, "learning_rate": 5e-05, "loss": 0.002930461196228862, "num_tokens": 5117656.0, "reward": 0.6524121165275574, "reward_std": 0.46509990096092224, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.047587890177965164, "rewards/length_penalty/std": 0.019898543134331703, "sampling/importance_sampling_ratio/max": 2.9863996505737305, "sampling/importance_sampling_ratio/mean": 0.9909448623657227, "sampling/importance_sampling_ratio/min": 0.4123409688472748, "sampling/sampling_logp_difference/max": 1.0940685272216797, "sampling/sampling_logp_difference/mean": 0.03773015737533569, "step": 95, "step_time": 2.79798654303886 }, { "clip_ratio/high_max": 0.0014799370197579264, "clip_ratio/high_mean": 0.0014799370197579264, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014799370197579264, "completions/clipped_ratio": 0.0, "completions/max_length": 551.0, "completions/max_terminated_length": 551.0, "completions/mean_length": 97.69999694824219, "completions/mean_terminated_length": 97.69999694824219, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.20367034077644347, "epoch": 0.2608695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.013027592562139034, "learning_rate": 5e-05, "loss": 0.01779348962008953, "num_tokens": 5126071.0, "reward": 0.552294909954071, "reward_std": 0.5121820569038391, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.04770507663488388, "rewards/length_penalty/std": 0.042042821645736694, "sampling/importance_sampling_ratio/max": 2.441518783569336, "sampling/importance_sampling_ratio/mean": 0.991175651550293, "sampling/importance_sampling_ratio/min": 0.06918129324913025, "sampling/sampling_logp_difference/max": 2.671024799346924, "sampling/sampling_logp_difference/mean": 0.03059057705104351, "step": 96, "step_time": 6.202032637782395 }, { "clip_ratio/high_max": 0.0011401220224797725, "clip_ratio/high_mean": 0.0011401220224797725, "clip_ratio/low_mean": 0.0009177979780361056, "clip_ratio/low_min": 0.0009177979780361056, "clip_ratio/region_mean": 0.002057919977232814, "completions/clipped_ratio": 0.0, "completions/max_length": 224.0, "completions/max_terminated_length": 224.0, "completions/mean_length": 86.97999572753906, "completions/mean_terminated_length": 86.97999572753906, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 0.2265522986650467, "epoch": 0.26358695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.01077208761125803, "learning_rate": 5e-05, "loss": 0.0009587033418938518, "num_tokens": 5133350.0, "reward": 0.4375292956829071, "reward_std": 0.5013757944107056, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.04247070476412773, "rewards/length_penalty/std": 0.021450260654091835, "sampling/importance_sampling_ratio/max": 2.6412837505340576, "sampling/importance_sampling_ratio/mean": 0.9908226728439331, "sampling/importance_sampling_ratio/min": 0.3037787675857544, "sampling/sampling_logp_difference/max": 1.191455602645874, "sampling/sampling_logp_difference/mean": 0.033219028264284134, "step": 97, "step_time": 3.065898902947083 }, { "clip_ratio/high_max": 0.0010683529078960418, "clip_ratio/high_mean": 0.0010683529078960418, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010683529078960418, "completions/clipped_ratio": 0.0, "completions/max_length": 150.0, "completions/max_terminated_length": 150.0, "completions/mean_length": 55.55999755859375, "completions/mean_terminated_length": 55.55999755859375, "completions/min_length": 18.0, "completions/min_terminated_length": 18.0, "entropy": 0.2746728152036667, "epoch": 0.266304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.012212988920509815, "learning_rate": 5e-05, "loss": 0.007884243503212929, "num_tokens": 5138268.0, "reward": 0.372871071100235, "reward_std": 0.5017322301864624, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.027128906920552254, "rewards/length_penalty/std": 0.014021570794284344, "sampling/importance_sampling_ratio/max": 2.170682907104492, "sampling/importance_sampling_ratio/mean": 0.9889690279960632, "sampling/importance_sampling_ratio/min": 0.35793039202690125, "sampling/sampling_logp_difference/max": 1.0274168252944946, "sampling/sampling_logp_difference/mean": 0.042866162955760956, "step": 98, "step_time": 2.4067599647678435 }, { "clip_ratio/high_max": 0.0008171118563041091, "clip_ratio/high_mean": 0.0008171118563041091, "clip_ratio/low_mean": 0.0007390069542452693, "clip_ratio/low_min": 0.0007390069542452693, "clip_ratio/region_mean": 0.0015561188105493785, "completions/clipped_ratio": 0.0, "completions/max_length": 94.0, "completions/max_terminated_length": 94.0, "completions/mean_length": 50.91999816894531, "completions/mean_terminated_length": 50.91999816894531, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 0.21310125291347504, "epoch": 0.26902173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.011825033463537693, "learning_rate": 5e-05, "loss": 0.0036260976921766996, "num_tokens": 5143724.0, "reward": 0.7351366877555847, "reward_std": 0.4356699287891388, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.02486328035593033, "rewards/length_penalty/std": 0.008906659670174122, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9978911876678467, "sampling/importance_sampling_ratio/min": 0.3821065127849579, "sampling/sampling_logp_difference/max": 1.398585319519043, "sampling/sampling_logp_difference/mean": 0.037688616663217545, "step": 99, "step_time": 2.0239664369728416 }, { "clip_ratio/high_max": 0.0019492614082992077, "clip_ratio/high_mean": 0.0019492614082992077, "clip_ratio/low_mean": 0.0007079645991325378, "clip_ratio/low_min": 0.0007079645991325378, "clip_ratio/region_mean": 0.0026572260074317457, "completions/clipped_ratio": 0.0, "completions/max_length": 125.0, "completions/max_terminated_length": 125.0, "completions/mean_length": 52.87999725341797, "completions/mean_terminated_length": 52.87999725341797, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 0.22497422993183136, "epoch": 0.2717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.01131928339600563, "learning_rate": 5e-05, "loss": 0.004754737485200167, "num_tokens": 5148598.0, "reward": 0.7741796970367432, "reward_std": 0.4040992259979248, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.025820313021540642, "rewards/length_penalty/std": 0.011249576695263386, "sampling/importance_sampling_ratio/max": 2.4267683029174805, "sampling/importance_sampling_ratio/mean": 0.9914778470993042, "sampling/importance_sampling_ratio/min": 0.27528116106987, "sampling/sampling_logp_difference/max": 1.2899622917175293, "sampling/sampling_logp_difference/mean": 0.03707539662718773, "step": 100, "step_time": 2.2108143279328942 } ], "logging_steps": 1, "max_steps": 200, "num_input_tokens_seen": 5148598, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 10, "trial_name": null, "trial_params": null }