{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.5434782608695652, "eval_steps": 500, "global_step": 200, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1671.2799072265625, "completions/mean_terminated_length": 1398.4827880859375, "completions/min_length": 992.0, "completions/min_terminated_length": 992.0, "entropy": 0.21248915791511536, "epoch": 0.002717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.01622791402041912, "learning_rate": 0.0, "loss": 0.06695985794067383, "num_tokens": 86304.0, "reward": -0.21605467796325684, "reward_std": 0.656523585319519, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.8160547018051147, "rewards/length_penalty/std": 0.18964654207229614, "sampling/importance_sampling_ratio/max": 1.5222556591033936, "sampling/importance_sampling_ratio/mean": 0.9927070140838623, "sampling/importance_sampling_ratio/min": 0.6435883045196533, "sampling/sampling_logp_difference/max": 0.44069600105285645, "sampling/sampling_logp_difference/mean": 0.013988969847559929, "step": 1, "step_time": 27.388843874214217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6800000071525574, "completions/max_length": 2048.0, "completions/max_terminated_length": 1991.0, "completions/mean_length": 1941.47998046875, "completions/mean_terminated_length": 1715.125, "completions/min_length": 1079.0, "completions/min_terminated_length": 1079.0, "entropy": 0.2905632793903351, "epoch": 0.005434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.017553681507706642, "learning_rate": 5e-06, "loss": 0.06798752397298813, "num_tokens": 186618.0, "reward": -0.7279882431030273, "reward_std": 0.4917677640914917, "rewards/correctness/mean": 0.2199999988079071, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.9479882717132568, "rewards/length_penalty/std": 0.10518474876880646, "sampling/importance_sampling_ratio/max": 1.450933814048767, "sampling/importance_sampling_ratio/mean": 0.9901063442230225, "sampling/importance_sampling_ratio/min": 0.3688630163669586, "sampling/sampling_logp_difference/max": 0.9973299503326416, "sampling/sampling_logp_difference/mean": 0.01780563034117222, "step": 2, "step_time": 29.02393381111324 }, { "clip_ratio/high_max": 0.0009701523347757756, "clip_ratio/high_mean": 0.0009701523347757756, "clip_ratio/low_mean": 6.219160495675169e-05, "clip_ratio/low_min": 6.219160495675169e-05, "clip_ratio/region_mean": 0.0010323439724743365, "completions/clipped_ratio": 0.3199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 1901.0, "completions/mean_length": 1612.3199462890625, "completions/mean_terminated_length": 1407.2940673828125, "completions/min_length": 745.0, "completions/min_terminated_length": 745.0, "entropy": 0.255071622133255, "epoch": 0.008152173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.017932165414094925, "learning_rate": 1e-05, "loss": 0.13346610963344574, "num_tokens": 269504.0, "reward": -0.08726562559604645, "reward_std": 0.6200677156448364, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.7872655987739563, "rewards/length_penalty/std": 0.19988179206848145, "sampling/importance_sampling_ratio/max": 1.5381298065185547, "sampling/importance_sampling_ratio/mean": 0.9911423921585083, "sampling/importance_sampling_ratio/min": 0.6395388841629028, "sampling/sampling_logp_difference/max": 0.4470078945159912, "sampling/sampling_logp_difference/mean": 0.016371745616197586, "step": 3, "step_time": 27.040046802023426 }, { "clip_ratio/high_max": 0.0007077743241097779, "clip_ratio/high_mean": 0.0007077743241097779, "clip_ratio/low_mean": 6.867761694593356e-05, "clip_ratio/low_min": 6.867761694593356e-05, "clip_ratio/region_mean": 0.0007764519308693707, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 1882.0, "completions/mean_length": 1489.419921875, "completions/mean_terminated_length": 1250.028564453125, "completions/min_length": 626.0, "completions/min_terminated_length": 626.0, "entropy": 0.24563565850257874, "epoch": 0.010869565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.017823319882154465, "learning_rate": 1.5e-05, "loss": 0.09811276942491531, "num_tokens": 346875.0, "reward": -0.007255859207361937, "reward_std": 0.6479107141494751, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.7272558808326721, "rewards/length_penalty/std": 0.2412886619567871, "sampling/importance_sampling_ratio/max": 1.5703521966934204, "sampling/importance_sampling_ratio/mean": 0.9914191961288452, "sampling/importance_sampling_ratio/min": 0.5496033430099487, "sampling/sampling_logp_difference/max": 0.5985584259033203, "sampling/sampling_logp_difference/mean": 0.016086140647530556, "step": 4, "step_time": 27.06473581609316 }, { "clip_ratio/high_max": 0.0014693949837237597, "clip_ratio/high_mean": 0.0014693949837237597, "clip_ratio/low_mean": 2.9735209682257847e-05, "clip_ratio/low_min": 2.9735209682257847e-05, "clip_ratio/region_mean": 0.0014991301926784218, "completions/clipped_ratio": 0.8199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 1909.0, "completions/mean_length": 1955.3800048828125, "completions/mean_terminated_length": 1533.4444580078125, "completions/min_length": 1097.0, "completions/min_terminated_length": 1097.0, "entropy": 0.2879446566104889, "epoch": 0.01358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.015369652770459652, "learning_rate": 2e-05, "loss": 0.05972106009721756, "num_tokens": 448354.0, "reward": -0.7747753858566284, "reward_std": 0.4882567822933197, "rewards/correctness/mean": 0.18000000715255737, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.9547753930091858, "rewards/length_penalty/std": 0.11000200361013412, "sampling/importance_sampling_ratio/max": 2.131683349609375, "sampling/importance_sampling_ratio/mean": 0.9901448488235474, "sampling/importance_sampling_ratio/min": 0.6602146625518799, "sampling/sampling_logp_difference/max": 0.7569119930267334, "sampling/sampling_logp_difference/mean": 0.01812317781150341, "step": 5, "step_time": 29.137280595256016 }, { "clip_ratio/high_max": 0.000520571404194925, "clip_ratio/high_mean": 0.000520571404194925, "clip_ratio/low_mean": 0.00017908125300891697, "clip_ratio/low_min": 0.00017908125300891697, "clip_ratio/region_mean": 0.0006996526441071183, "completions/clipped_ratio": 0.4599999785423279, "completions/max_length": 2048.0, "completions/max_terminated_length": 1941.0, "completions/mean_length": 1660.0, "completions/mean_terminated_length": 1329.4814453125, "completions/min_length": 723.0, "completions/min_terminated_length": 723.0, "entropy": 0.2539885014295578, "epoch": 0.016304347826086956, "frac_reward_zero_std": 0.0, "grad_norm": 0.01736672781407833, "learning_rate": 2.5e-05, "loss": 0.08885529637336731, "num_tokens": 534344.0, "reward": -0.23054687678813934, "reward_std": 0.6794751286506653, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.810546875, "rewards/length_penalty/std": 0.22522568702697754, "sampling/importance_sampling_ratio/max": 1.742319941520691, "sampling/importance_sampling_ratio/mean": 0.9909853339195251, "sampling/importance_sampling_ratio/min": 0.5412515997886658, "sampling/sampling_logp_difference/max": 0.6138710975646973, "sampling/sampling_logp_difference/mean": 0.016689306125044823, "step": 6, "step_time": 27.337108821840957 }, { "clip_ratio/high_max": 0.0006314931088127196, "clip_ratio/high_mean": 0.0006314931088127196, "clip_ratio/low_mean": 7.634440262336284e-05, "clip_ratio/low_min": 7.634440262336284e-05, "clip_ratio/region_mean": 0.0007078375143464655, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 2029.0, "completions/mean_length": 1579.8399658203125, "completions/mean_terminated_length": 1379.199951171875, "completions/min_length": 803.0, "completions/min_terminated_length": 803.0, "entropy": 0.2385493516921997, "epoch": 0.019021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.016621841117739677, "learning_rate": 3e-05, "loss": 0.08834289014339447, "num_tokens": 615566.0, "reward": -0.05140624940395355, "reward_std": 0.6118109226226807, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.7714062333106995, "rewards/length_penalty/std": 0.19486749172210693, "sampling/importance_sampling_ratio/max": 1.4593793153762817, "sampling/importance_sampling_ratio/mean": 0.9915305972099304, "sampling/importance_sampling_ratio/min": 0.651093065738678, "sampling/sampling_logp_difference/max": 0.42910265922546387, "sampling/sampling_logp_difference/mean": 0.015702536329627037, "step": 7, "step_time": 27.038635928882286 }, { "clip_ratio/high_max": 0.0008715830335859209, "clip_ratio/high_mean": 0.0008715830335859209, "clip_ratio/low_mean": 0.00011999263515463098, "clip_ratio/low_min": 0.00011999263515463098, "clip_ratio/region_mean": 0.0009915756527334452, "completions/clipped_ratio": 0.5999999642372131, "completions/max_length": 2048.0, "completions/max_terminated_length": 1938.0, "completions/mean_length": 1801.239990234375, "completions/mean_terminated_length": 1431.0999755859375, "completions/min_length": 930.0, "completions/min_terminated_length": 930.0, "entropy": 0.21174302995204924, "epoch": 0.021739130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.01676737703382969, "learning_rate": 3.5e-05, "loss": 0.08332858979701996, "num_tokens": 707778.0, "reward": -0.47951170802116394, "reward_std": 0.6519015431404114, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.8795117139816284, "rewards/length_penalty/std": 0.18039129674434662, "sampling/importance_sampling_ratio/max": 1.4509235620498657, "sampling/importance_sampling_ratio/mean": 0.9925462007522583, "sampling/importance_sampling_ratio/min": 0.052912116050720215, "sampling/sampling_logp_difference/max": 2.9391229152679443, "sampling/sampling_logp_difference/mean": 0.014070631936192513, "step": 8, "step_time": 28.223328100983053 }, { "clip_ratio/high_max": 0.0006956349650863558, "clip_ratio/high_mean": 0.0006956349650863558, "clip_ratio/low_mean": 8.642674874863588e-05, "clip_ratio/low_min": 8.642674874863588e-05, "clip_ratio/region_mean": 0.0007820617058314383, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1994.0, "completions/mean_length": 1600.239990234375, "completions/mean_terminated_length": 1442.9189453125, "completions/min_length": 785.0, "completions/min_terminated_length": 785.0, "entropy": 0.23202865421772004, "epoch": 0.024456521739130436, "frac_reward_zero_std": 0.0, "grad_norm": 0.017678245902061462, "learning_rate": 4e-05, "loss": 0.09870695322751999, "num_tokens": 790260.0, "reward": -0.20136718451976776, "reward_std": 0.6241466403007507, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.7813671827316284, "rewards/length_penalty/std": 0.1977044939994812, "sampling/importance_sampling_ratio/max": 1.5457746982574463, "sampling/importance_sampling_ratio/mean": 0.991839587688446, "sampling/importance_sampling_ratio/min": 0.556106448173523, "sampling/sampling_logp_difference/max": 0.5867955684661865, "sampling/sampling_logp_difference/mean": 0.01518256589770317, "step": 9, "step_time": 27.120734592201188 }, { "clip_ratio/high_max": 0.0007978752721101046, "clip_ratio/high_mean": 0.0007978752721101046, "clip_ratio/low_mean": 7.057307666400448e-05, "clip_ratio/low_min": 7.057307666400448e-05, "clip_ratio/region_mean": 0.0008684483473189175, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 2048.0, "completions/mean_length": 1484.0999755859375, "completions/mean_terminated_length": 1264.8055419921875, "completions/min_length": 668.0, "completions/min_terminated_length": 668.0, "entropy": 0.24629398584365844, "epoch": 0.02717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.017371023073792458, "learning_rate": 4.5e-05, "loss": 0.09812771528959274, "num_tokens": 869285.0, "reward": -0.024658203125, "reward_std": 0.647922933101654, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.724658191204071, "rewards/length_penalty/std": 0.2301420420408249, "sampling/importance_sampling_ratio/max": 1.4670653343200684, "sampling/importance_sampling_ratio/mean": 0.9915652275085449, "sampling/importance_sampling_ratio/min": 0.6655772924423218, "sampling/sampling_logp_difference/max": 0.4071004390716553, "sampling/sampling_logp_difference/mean": 0.015878628939390182, "step": 10, "step_time": 27.66670753993094 }, { "clip_ratio/high_max": 0.0007490424031857401, "clip_ratio/high_mean": 0.0007490424031857401, "clip_ratio/low_mean": 0.00011397197667974978, "clip_ratio/low_min": 0.00011397197667974978, "clip_ratio/region_mean": 0.0008630143827758729, "completions/clipped_ratio": 0.47999998927116394, "completions/max_length": 2048.0, "completions/max_terminated_length": 2034.0, "completions/mean_length": 1854.7999267578125, "completions/mean_terminated_length": 1676.4615478515625, "completions/min_length": 1178.0, "completions/min_terminated_length": 1178.0, "entropy": 0.24558066129684447, "epoch": 0.029891304347826088, "frac_reward_zero_std": 0.0, "grad_norm": 0.01811002753674984, "learning_rate": 5e-05, "loss": 0.08611460030078888, "num_tokens": 964445.0, "reward": -0.32566404342651367, "reward_std": 0.5863887667655945, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.9056640863418579, "rewards/length_penalty/std": 0.12008737027645111, "sampling/importance_sampling_ratio/max": 1.515191912651062, "sampling/importance_sampling_ratio/mean": 0.9914591312408447, "sampling/importance_sampling_ratio/min": 0.2949647009372711, "sampling/sampling_logp_difference/max": 1.2208995819091797, "sampling/sampling_logp_difference/mean": 0.0158162210136652, "step": 11, "step_time": 28.245034573134035 }, { "clip_ratio/high_max": 0.0004984279570635408, "clip_ratio/high_mean": 0.0004984279570635408, "clip_ratio/low_mean": 0.00014867318095639347, "clip_ratio/low_min": 0.00014867318095639347, "clip_ratio/region_mean": 0.0006471011089161039, "completions/clipped_ratio": 0.5, "completions/max_length": 2048.0, "completions/max_terminated_length": 2027.0, "completions/mean_length": 1772.43994140625, "completions/mean_terminated_length": 1496.8800048828125, "completions/min_length": 1000.0, "completions/min_terminated_length": 1000.0, "entropy": 0.2841207951307297, "epoch": 0.03260869565217391, "frac_reward_zero_std": 0.0, "grad_norm": 0.018444733694195747, "learning_rate": 5e-05, "loss": 0.10594244301319122, "num_tokens": 1055707.0, "reward": -0.3654492199420929, "reward_std": 0.6480823159217834, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.8654491901397705, "rewards/length_penalty/std": 0.1661682426929474, "sampling/importance_sampling_ratio/max": 1.595076084136963, "sampling/importance_sampling_ratio/mean": 0.9902372360229492, "sampling/importance_sampling_ratio/min": 0.6496405005455017, "sampling/sampling_logp_difference/max": 0.46692144870758057, "sampling/sampling_logp_difference/mean": 0.018158787861466408, "step": 12, "step_time": 28.327091000042856 }, { "clip_ratio/high_max": 0.0005573231639573351, "clip_ratio/high_mean": 0.0005573231639573351, "clip_ratio/low_mean": 4.4240543502382934e-05, "clip_ratio/low_min": 4.4240543502382934e-05, "clip_ratio/region_mean": 0.0006015637016389519, "completions/clipped_ratio": 0.3999999761581421, "completions/max_length": 2048.0, "completions/max_terminated_length": 1992.0, "completions/mean_length": 1732.1199951171875, "completions/mean_terminated_length": 1521.533447265625, "completions/min_length": 840.0, "completions/min_terminated_length": 840.0, "entropy": 0.25946328043937683, "epoch": 0.035326086956521736, "frac_reward_zero_std": 0.0, "grad_norm": 0.01701098121702671, "learning_rate": 5e-05, "loss": 0.07992903143167496, "num_tokens": 1145333.0, "reward": -0.40576171875, "reward_std": 0.6230165958404541, "rewards/correctness/mean": 0.4399999976158142, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.8457617163658142, "rewards/length_penalty/std": 0.18863798677921295, "sampling/importance_sampling_ratio/max": 1.4509283304214478, "sampling/importance_sampling_ratio/mean": 0.9908791780471802, "sampling/importance_sampling_ratio/min": 0.6448545455932617, "sampling/sampling_logp_difference/max": 0.43873047828674316, "sampling/sampling_logp_difference/mean": 0.016880951821804047, "step": 13, "step_time": 27.769306767731905 }, { "clip_ratio/high_max": 0.000875298772007227, "clip_ratio/high_mean": 0.000875298772007227, "clip_ratio/low_mean": 0.00014390671713044868, "clip_ratio/low_min": 0.00014390671713044868, "clip_ratio/region_mean": 0.001019205478951335, "completions/clipped_ratio": 0.47999998927116394, "completions/max_length": 2048.0, "completions/max_terminated_length": 2015.0, "completions/mean_length": 1752.8399658203125, "completions/mean_terminated_length": 1480.3846435546875, "completions/min_length": 975.0, "completions/min_terminated_length": 975.0, "entropy": 0.2478518307209015, "epoch": 0.03804347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.017872340977191925, "learning_rate": 5e-05, "loss": 0.10336586833000183, "num_tokens": 1235535.0, "reward": -0.3358789086341858, "reward_std": 0.654521644115448, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.8558788895606995, "rewards/length_penalty/std": 0.18034636974334717, "sampling/importance_sampling_ratio/max": 1.4315266609191895, "sampling/importance_sampling_ratio/mean": 0.9911905527114868, "sampling/importance_sampling_ratio/min": 0.5119919180870056, "sampling/sampling_logp_difference/max": 0.6694464683532715, "sampling/sampling_logp_difference/mean": 0.016088664531707764, "step": 14, "step_time": 28.357874361332506 }, { "clip_ratio/high_max": 0.0006910582887940109, "clip_ratio/high_mean": 0.0006910582887940109, "clip_ratio/low_mean": 8.928978495532647e-05, "clip_ratio/low_min": 8.928978495532647e-05, "clip_ratio/region_mean": 0.0007803480722941459, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 1903.0, "completions/mean_length": 1541.0399169921875, "completions/mean_terminated_length": 1343.888916015625, "completions/min_length": 961.0, "completions/min_terminated_length": 961.0, "entropy": 0.17789481580257416, "epoch": 0.04076086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.019630515947937965, "learning_rate": 5e-05, "loss": 0.0840536430478096, "num_tokens": 1315167.0, "reward": -0.2124609351158142, "reward_std": 0.5916866064071655, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.7524609565734863, "rewards/length_penalty/std": 0.18993473052978516, "sampling/importance_sampling_ratio/max": 1.4382117986679077, "sampling/importance_sampling_ratio/mean": 0.9938375353813171, "sampling/importance_sampling_ratio/min": 0.6220142841339111, "sampling/sampling_logp_difference/max": 0.4747922420501709, "sampling/sampling_logp_difference/mean": 0.011926891282200813, "step": 15, "step_time": 27.46101871319115 }, { "clip_ratio/high_max": 0.0012169468565844, "clip_ratio/high_mean": 0.0012169468565844, "clip_ratio/low_mean": 6.13222160609439e-05, "clip_ratio/low_min": 6.13222160609439e-05, "clip_ratio/region_mean": 0.0012782690697349607, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 1896.0, "completions/mean_length": 1608.760009765625, "completions/mean_terminated_length": 1290.689697265625, "completions/min_length": 763.0, "completions/min_terminated_length": 763.0, "entropy": 0.25876877307891843, "epoch": 0.043478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.01870792917907238, "learning_rate": 5e-05, "loss": 0.10105059295892715, "num_tokens": 1399705.0, "reward": -0.18552733957767487, "reward_std": 0.6857022643089294, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.7855273485183716, "rewards/length_penalty/std": 0.22407564520835876, "sampling/importance_sampling_ratio/max": 1.8998873233795166, "sampling/importance_sampling_ratio/mean": 0.990841805934906, "sampling/importance_sampling_ratio/min": 0.5615923404693604, "sampling/sampling_logp_difference/max": 0.6417945623397827, "sampling/sampling_logp_difference/mean": 0.016635797917842865, "step": 16, "step_time": 27.157737250905484 }, { "clip_ratio/high_max": 0.0006164561185869388, "clip_ratio/high_mean": 0.0006164561185869388, "clip_ratio/low_mean": 0.0001622869363927748, "clip_ratio/low_min": 0.0001622869363927748, "clip_ratio/region_mean": 0.0007787430338794365, "completions/clipped_ratio": 0.47999998927116394, "completions/max_length": 2048.0, "completions/max_terminated_length": 2028.0, "completions/mean_length": 1708.8399658203125, "completions/mean_terminated_length": 1395.769287109375, "completions/min_length": 868.0, "completions/min_terminated_length": 868.0, "entropy": 0.2281492084264755, "epoch": 0.04619565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.01940748654305935, "learning_rate": 5e-05, "loss": 0.07547423243522644, "num_tokens": 1487797.0, "reward": -0.3143945336341858, "reward_std": 0.6761555075645447, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.8343945145606995, "rewards/length_penalty/std": 0.20066553354263306, "sampling/importance_sampling_ratio/max": 1.667822241783142, "sampling/importance_sampling_ratio/mean": 0.9918660521507263, "sampling/importance_sampling_ratio/min": 0.5456497669219971, "sampling/sampling_logp_difference/max": 0.6057779788970947, "sampling/sampling_logp_difference/mean": 0.01505908090621233, "step": 17, "step_time": 27.55593749205582 }, { "clip_ratio/high_max": 0.0007688433746807277, "clip_ratio/high_mean": 0.0007688433746807277, "clip_ratio/low_mean": 6.974122588871978e-05, "clip_ratio/low_min": 6.974122588871978e-05, "clip_ratio/region_mean": 0.0008385846158489585, "completions/clipped_ratio": 0.23999999463558197, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 1681.239990234375, "completions/mean_terminated_length": 1565.4210205078125, "completions/min_length": 1013.0, "completions/min_terminated_length": 1013.0, "entropy": 0.2069709062576294, "epoch": 0.04891304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.01836375705897808, "learning_rate": 5e-05, "loss": 0.07770835608243942, "num_tokens": 1574359.0, "reward": -0.20091795921325684, "reward_std": 0.6101290583610535, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.8209179639816284, "rewards/length_penalty/std": 0.1540166735649109, "sampling/importance_sampling_ratio/max": 1.4398912191390991, "sampling/importance_sampling_ratio/mean": 0.9928213357925415, "sampling/importance_sampling_ratio/min": 0.6220183372497559, "sampling/sampling_logp_difference/max": 0.4747856855392456, "sampling/sampling_logp_difference/mean": 0.013566325418651104, "step": 18, "step_time": 27.951239524874836 }, { "clip_ratio/high_max": 0.0014653601916506886, "clip_ratio/high_mean": 0.0014653601916506886, "clip_ratio/low_mean": 0.00016244076541624964, "clip_ratio/low_min": 0.00016244076541624964, "clip_ratio/region_mean": 0.0016278009628877045, "completions/clipped_ratio": 0.5600000023841858, "completions/max_length": 2048.0, "completions/max_terminated_length": 2028.0, "completions/mean_length": 1752.8199462890625, "completions/mean_terminated_length": 1377.1363525390625, "completions/min_length": 589.0, "completions/min_terminated_length": 589.0, "entropy": 0.2843230068683624, "epoch": 0.051630434782608696, "frac_reward_zero_std": 0.0, "grad_norm": 0.019417675212025642, "learning_rate": 5e-05, "loss": 0.0901256799697876, "num_tokens": 1665430.0, "reward": -0.3958691358566284, "reward_std": 0.6787852048873901, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.8558691143989563, "rewards/length_penalty/std": 0.22009503841400146, "sampling/importance_sampling_ratio/max": 1.4385100603103638, "sampling/importance_sampling_ratio/mean": 0.9899238348007202, "sampling/importance_sampling_ratio/min": 0.5758324265480042, "sampling/sampling_logp_difference/max": 0.5519386529922485, "sampling/sampling_logp_difference/mean": 0.0180143304169178, "step": 19, "step_time": 28.49813059018925 }, { "clip_ratio/high_max": 0.0012508025160059334, "clip_ratio/high_mean": 0.0012508025160059334, "clip_ratio/low_mean": 5.348440172383562e-05, "clip_ratio/low_min": 5.348440172383562e-05, "clip_ratio/region_mean": 0.0013042868813499808, "completions/clipped_ratio": 0.5199999809265137, "completions/max_length": 2048.0, "completions/max_terminated_length": 2046.0, "completions/mean_length": 1844.43994140625, "completions/mean_terminated_length": 1623.916748046875, "completions/min_length": 1120.0, "completions/min_terminated_length": 1120.0, "entropy": 0.2606961756944656, "epoch": 0.05434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.01755714789032936, "learning_rate": 5e-05, "loss": 0.09018535166978836, "num_tokens": 1760242.0, "reward": -0.42060545086860657, "reward_std": 0.6140408515930176, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.9006054401397705, "rewards/length_penalty/std": 0.12989383935928345, "sampling/importance_sampling_ratio/max": 1.3885498046875, "sampling/importance_sampling_ratio/mean": 0.9910284280776978, "sampling/importance_sampling_ratio/min": 0.4727957248687744, "sampling/sampling_logp_difference/max": 0.7490918636322021, "sampling/sampling_logp_difference/mean": 0.01643785461783409, "step": 20, "step_time": 27.96463798894547 }, { "clip_ratio/high_max": 0.0007326291175559163, "clip_ratio/high_mean": 0.0007326291175559163, "clip_ratio/low_mean": 0.00013991183222969994, "clip_ratio/low_min": 0.00013991183222969994, "clip_ratio/region_mean": 0.0008725409512408078, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 2016.0, "completions/mean_length": 1689.5599365234375, "completions/mean_terminated_length": 1535.94287109375, "completions/min_length": 1011.0, "completions/min_terminated_length": 1011.0, "entropy": 0.2159762352705002, "epoch": 0.057065217391304345, "frac_reward_zero_std": 0.0, "grad_norm": 0.016861431300640106, "learning_rate": 5e-05, "loss": 0.08257238566875458, "num_tokens": 1848130.0, "reward": -0.12498046457767487, "reward_std": 0.5931032299995422, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.8249804973602295, "rewards/length_penalty/std": 0.17415757477283478, "sampling/importance_sampling_ratio/max": 1.4638551473617554, "sampling/importance_sampling_ratio/mean": 0.9920001029968262, "sampling/importance_sampling_ratio/min": 0.5236321091651917, "sampling/sampling_logp_difference/max": 0.6469658613204956, "sampling/sampling_logp_difference/mean": 0.014182405546307564, "step": 21, "step_time": 26.797411655075848 }, { "clip_ratio/high_max": 0.0011293109448160977, "clip_ratio/high_mean": 0.0011293109448160977, "clip_ratio/low_mean": 9.711003804113716e-05, "clip_ratio/low_min": 9.711003804113716e-05, "clip_ratio/region_mean": 0.0012264209799468517, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 1946.0, "completions/mean_length": 1501.0599365234375, "completions/mean_terminated_length": 1288.361083984375, "completions/min_length": 687.0, "completions/min_terminated_length": 687.0, "entropy": 0.2569431811571121, "epoch": 0.059782608695652176, "frac_reward_zero_std": 0.0, "grad_norm": 0.020697707310318947, "learning_rate": 5e-05, "loss": 0.12516915798187256, "num_tokens": 1928253.0, "reward": 0.0070605468936264515, "reward_std": 0.6210074424743652, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.7329394817352295, "rewards/length_penalty/std": 0.21821558475494385, "sampling/importance_sampling_ratio/max": 1.6245068311691284, "sampling/importance_sampling_ratio/mean": 0.9910734295845032, "sampling/importance_sampling_ratio/min": 0.5689723491668701, "sampling/sampling_logp_difference/max": 0.5639233589172363, "sampling/sampling_logp_difference/mean": 0.016499562188982964, "step": 22, "step_time": 27.23643856518902 }, { "clip_ratio/high_max": 0.0009234562981873751, "clip_ratio/high_mean": 0.0009234562981873751, "clip_ratio/low_mean": 0.00011172640952281654, "clip_ratio/low_min": 0.00011172640952281654, "clip_ratio/region_mean": 0.0010351826902478933, "completions/clipped_ratio": 0.3400000035762787, "completions/max_length": 2048.0, "completions/max_terminated_length": 1951.0, "completions/mean_length": 1656.260009765625, "completions/mean_terminated_length": 1454.45458984375, "completions/min_length": 875.0, "completions/min_terminated_length": 875.0, "entropy": 0.2721762388944626, "epoch": 0.0625, "frac_reward_zero_std": 0.0, "grad_norm": 0.02058061957359314, "learning_rate": 5e-05, "loss": 0.1155262216925621, "num_tokens": 2017186.0, "reward": -0.2487206906080246, "reward_std": 0.6356157064437866, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.8087207078933716, "rewards/length_penalty/std": 0.1684350073337555, "sampling/importance_sampling_ratio/max": 1.5564347505569458, "sampling/importance_sampling_ratio/mean": 0.9903766512870789, "sampling/importance_sampling_ratio/min": 0.5593669414520264, "sampling/sampling_logp_difference/max": 0.5809495449066162, "sampling/sampling_logp_difference/mean": 0.01754668541252613, "step": 23, "step_time": 27.693883403902873 }, { "clip_ratio/high_max": 0.0005631370135233737, "clip_ratio/high_mean": 0.0005631370135233737, "clip_ratio/low_mean": 0.00010890306512010284, "clip_ratio/low_min": 0.00010890306512010284, "clip_ratio/region_mean": 0.0006720400706399232, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 2034.0, "completions/mean_length": 1659.1199951171875, "completions/mean_terminated_length": 1377.5172119140625, "completions/min_length": 734.0, "completions/min_terminated_length": 734.0, "entropy": 0.2400749534368515, "epoch": 0.06521739130434782, "frac_reward_zero_std": 0.0, "grad_norm": 0.017921488732099533, "learning_rate": 5e-05, "loss": 0.0665205717086792, "num_tokens": 2102432.0, "reward": -0.2701171934604645, "reward_std": 0.675529420375824, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.8101171851158142, "rewards/length_penalty/std": 0.22776594758033752, "sampling/importance_sampling_ratio/max": 1.6843202114105225, "sampling/importance_sampling_ratio/mean": 0.9915715456008911, "sampling/importance_sampling_ratio/min": 0.4471953809261322, "sampling/sampling_logp_difference/max": 0.8047597408294678, "sampling/sampling_logp_difference/mean": 0.01562676392495632, "step": 24, "step_time": 27.030204062117264 }, { "clip_ratio/high_max": 0.0010878795757889748, "clip_ratio/high_mean": 0.0010878795757889748, "clip_ratio/low_mean": 0.00010777307761600241, "clip_ratio/low_min": 0.00010777307761600241, "clip_ratio/region_mean": 0.0011956526315771044, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 2004.0, "completions/mean_length": 1457.0599365234375, "completions/mean_terminated_length": 1290.3846435546875, "completions/min_length": 754.0, "completions/min_terminated_length": 754.0, "entropy": 0.28386659622192384, "epoch": 0.06793478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.021549172699451447, "learning_rate": 5e-05, "loss": 0.10626962780952454, "num_tokens": 2177685.0, "reward": 0.04854492098093033, "reward_std": 0.6079964637756348, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.7114551067352295, "rewards/length_penalty/std": 0.21497680246829987, "sampling/importance_sampling_ratio/max": 1.7750171422958374, "sampling/importance_sampling_ratio/mean": 0.9901482462882996, "sampling/importance_sampling_ratio/min": 0.3599497675895691, "sampling/sampling_logp_difference/max": 1.021790862083435, "sampling/sampling_logp_difference/mean": 0.018604418262839317, "step": 25, "step_time": 25.64425123319961 }, { "clip_ratio/high_max": 0.000790244567906484, "clip_ratio/high_mean": 0.000790244567906484, "clip_ratio/low_mean": 4.310927906772122e-05, "clip_ratio/low_min": 4.310927906772122e-05, "clip_ratio/region_mean": 0.0008333538484293967, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 2031.0, "completions/mean_length": 1346.1199951171875, "completions/mean_terminated_length": 1285.0870361328125, "completions/min_length": 729.0, "completions/min_terminated_length": 729.0, "entropy": 0.2296551436185837, "epoch": 0.07065217391304347, "frac_reward_zero_std": 0.0, "grad_norm": 0.020984938368201256, "learning_rate": 5e-05, "loss": 0.09681613743305206, "num_tokens": 2247461.0, "reward": -0.09728515148162842, "reward_std": 0.5121884346008301, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.6572851538658142, "rewards/length_penalty/std": 0.19092224538326263, "sampling/importance_sampling_ratio/max": 1.9684982299804688, "sampling/importance_sampling_ratio/mean": 0.9919283986091614, "sampling/importance_sampling_ratio/min": 0.34581977128982544, "sampling/sampling_logp_difference/max": 1.0618375539779663, "sampling/sampling_logp_difference/mean": 0.015773361548781395, "step": 26, "step_time": 25.79908507759683 }, { "clip_ratio/high_max": 0.0005374234810005873, "clip_ratio/high_mean": 0.0005374234810005873, "clip_ratio/low_mean": 0.00010813681001309305, "clip_ratio/low_min": 0.00010813681001309305, "clip_ratio/region_mean": 0.0006455602881032974, "completions/clipped_ratio": 0.3999999761581421, "completions/max_length": 2048.0, "completions/max_terminated_length": 1926.0, "completions/mean_length": 1517.8599853515625, "completions/mean_terminated_length": 1164.433349609375, "completions/min_length": 606.0, "completions/min_terminated_length": 606.0, "entropy": 0.2029076486825943, "epoch": 0.07336956521739131, "frac_reward_zero_std": 0.0, "grad_norm": 0.017958836629986763, "learning_rate": 5e-05, "loss": 0.028417475521564484, "num_tokens": 2325604.0, "reward": -0.12114257365465164, "reward_std": 0.709344208240509, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.7411425709724426, "rewards/length_penalty/std": 0.2490270882844925, "sampling/importance_sampling_ratio/max": 1.7574549913406372, "sampling/importance_sampling_ratio/mean": 0.9929352402687073, "sampling/importance_sampling_ratio/min": 0.2425677329301834, "sampling/sampling_logp_difference/max": 1.4164743423461914, "sampling/sampling_logp_difference/mean": 0.013834730722010136, "step": 27, "step_time": 25.780867578927428 }, { "clip_ratio/high_max": 0.0008416285156272352, "clip_ratio/high_mean": 0.0008416285156272352, "clip_ratio/low_mean": 4.83594267279841e-05, "clip_ratio/low_min": 4.83594267279841e-05, "clip_ratio/region_mean": 0.0008899879525415599, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 1903.0, "completions/mean_length": 1254.239990234375, "completions/mean_terminated_length": 1203.574462890625, "completions/min_length": 796.0, "completions/min_terminated_length": 796.0, "entropy": 0.18921910226345062, "epoch": 0.07608695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.019402744248509407, "learning_rate": 5e-05, "loss": 0.108873650431633, "num_tokens": 2391676.0, "reward": 0.12757812440395355, "reward_std": 0.5219429731369019, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308748841285706, "rewards/length_penalty/mean": -0.6124218702316284, "rewards/length_penalty/std": 0.16508276760578156, "sampling/importance_sampling_ratio/max": 1.7898136377334595, "sampling/importance_sampling_ratio/mean": 0.9934007525444031, "sampling/importance_sampling_ratio/min": 0.2999454140663147, "sampling/sampling_logp_difference/max": 1.2041547298431396, "sampling/sampling_logp_difference/mean": 0.013383659534156322, "step": 28, "step_time": 25.343337242724374 }, { "clip_ratio/high_max": 0.0007759740750771016, "clip_ratio/high_mean": 0.0007759740750771016, "clip_ratio/low_mean": 8.760135315242224e-05, "clip_ratio/low_min": 8.760135315242224e-05, "clip_ratio/region_mean": 0.0008635754231363535, "completions/clipped_ratio": 0.3799999952316284, "completions/max_length": 2048.0, "completions/max_terminated_length": 2005.0, "completions/mean_length": 1616.659912109375, "completions/mean_terminated_length": 1352.290283203125, "completions/min_length": 575.0, "completions/min_terminated_length": 575.0, "entropy": 0.24235570728778838, "epoch": 0.07880434782608696, "frac_reward_zero_std": 0.0, "grad_norm": 0.019807059317827225, "learning_rate": 5e-05, "loss": 0.06528324633836746, "num_tokens": 2475949.0, "reward": -0.12938475608825684, "reward_std": 0.6638873219490051, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.47851812839508057, "rewards/length_penalty/mean": -0.7893847823143005, "rewards/length_penalty/std": 0.25617480278015137, "sampling/importance_sampling_ratio/max": 1.7938244342803955, "sampling/importance_sampling_ratio/mean": 0.9914634823799133, "sampling/importance_sampling_ratio/min": 0.3581945598125458, "sampling/sampling_logp_difference/max": 1.0266790390014648, "sampling/sampling_logp_difference/mean": 0.01648593135178089, "step": 29, "step_time": 26.831963051110506 }, { "clip_ratio/high_max": 0.0007342056138440967, "clip_ratio/high_mean": 0.0007342056138440967, "clip_ratio/low_mean": 6.386294553522021e-05, "clip_ratio/low_min": 6.386294553522021e-05, "clip_ratio/region_mean": 0.000798068568110466, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1963.0, "completions/mean_length": 1230.659912109375, "completions/mean_terminated_length": 1139.844482421875, "completions/min_length": 761.0, "completions/min_terminated_length": 761.0, "entropy": 0.1697321891784668, "epoch": 0.08152173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.015965549275279045, "learning_rate": 5e-05, "loss": 0.07354339957237244, "num_tokens": 2539272.0, "reward": 0.319091796875, "reward_std": 0.4232766628265381, "rewards/correctness/mean": 0.9200000166893005, "rewards/correctness/std": 0.27404749393463135, "rewards/length_penalty/mean": -0.6009082198143005, "rewards/length_penalty/std": 0.19723902642726898, "sampling/importance_sampling_ratio/max": 1.9091259241104126, "sampling/importance_sampling_ratio/mean": 0.9938931465148926, "sampling/importance_sampling_ratio/min": 0.006599857937544584, "sampling/sampling_logp_difference/max": 5.020707130432129, "sampling/sampling_logp_difference/mean": 0.012903784401714802, "step": 30, "step_time": 24.605497697601095 }, { "clip_ratio/high_max": 0.0005373104766476899, "clip_ratio/high_mean": 0.0005373104766476899, "clip_ratio/low_mean": 0.00013891156122554095, "clip_ratio/low_min": 0.00013891156122554095, "clip_ratio/region_mean": 0.0006762220233213157, "completions/clipped_ratio": 0.3400000035762787, "completions/max_length": 2048.0, "completions/max_terminated_length": 1972.0, "completions/mean_length": 1636.5999755859375, "completions/mean_terminated_length": 1424.666748046875, "completions/min_length": 671.0, "completions/min_terminated_length": 671.0, "entropy": 0.18908907771110534, "epoch": 0.08423913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.01748133823275566, "learning_rate": 5e-05, "loss": 0.05618336424231529, "num_tokens": 2624232.0, "reward": -0.4791210889816284, "reward_std": 0.46296438574790955, "rewards/correctness/mean": 0.3199999928474426, "rewards/correctness/std": 0.4712120592594147, "rewards/length_penalty/mean": -0.799121081829071, "rewards/length_penalty/std": 0.2133444994688034, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.993071436882019, "sampling/importance_sampling_ratio/min": 0.3394291400909424, "sampling/sampling_logp_difference/max": 1.2877426147460938, "sampling/sampling_logp_difference/mean": 0.013400268740952015, "step": 31, "step_time": 27.240520113147795 }, { "clip_ratio/high_max": 0.0011355961789377034, "clip_ratio/high_mean": 0.0011355961789377034, "clip_ratio/low_mean": 0.00015650656423531472, "clip_ratio/low_min": 0.00015650656423531472, "clip_ratio/region_mean": 0.0012921027606353163, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 2022.0, "completions/mean_length": 1262.5599365234375, "completions/mean_terminated_length": 1155.45458984375, "completions/min_length": 595.0, "completions/min_terminated_length": 595.0, "entropy": 0.22945527732372284, "epoch": 0.08695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.01773648150265217, "learning_rate": 5e-05, "loss": 0.09414535760879517, "num_tokens": 2689290.0, "reward": 0.2235156148672104, "reward_std": 0.5308910012245178, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.6164844036102295, "rewards/length_penalty/std": 0.2093491554260254, "sampling/importance_sampling_ratio/max": 2.3375766277313232, "sampling/importance_sampling_ratio/mean": 0.9917176961898804, "sampling/importance_sampling_ratio/min": 0.30112117528915405, "sampling/sampling_logp_difference/max": 1.200242519378662, "sampling/sampling_logp_difference/mean": 0.01688900962471962, "step": 32, "step_time": 24.735927454195917 }, { "clip_ratio/high_max": 0.000496267294511199, "clip_ratio/high_mean": 0.000496267294511199, "clip_ratio/low_mean": 0.00010696919343899935, "clip_ratio/low_min": 0.00010696919343899935, "clip_ratio/region_mean": 0.0006032364792190493, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1985.0, "completions/mean_length": 1416.780029296875, "completions/mean_terminated_length": 1195.0, "completions/min_length": 765.0, "completions/min_terminated_length": 765.0, "entropy": 0.19317569732666015, "epoch": 0.08967391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.01730223558843136, "learning_rate": 5e-05, "loss": 0.029047608375549316, "num_tokens": 2763769.0, "reward": -0.09178710728883743, "reward_std": 0.6523070335388184, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716163635254, "rewards/length_penalty/mean": -0.6917871236801147, "rewards/length_penalty/std": 0.2441810518503189, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930633902549744, "sampling/importance_sampling_ratio/min": 0.23537996411323547, "sampling/sampling_logp_difference/max": 1.446554183959961, "sampling/sampling_logp_difference/mean": 0.014113553799688816, "step": 33, "step_time": 26.25649970700033 }, { "clip_ratio/high_max": 0.0010051162447780372, "clip_ratio/high_mean": 0.0010051162447780372, "clip_ratio/low_mean": 7.109031575964763e-05, "clip_ratio/low_min": 7.109031575964763e-05, "clip_ratio/region_mean": 0.0010762065765447914, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 1776.0, "completions/mean_length": 1481.6400146484375, "completions/mean_terminated_length": 1238.914306640625, "completions/min_length": 812.0, "completions/min_terminated_length": 812.0, "entropy": 0.23732842803001403, "epoch": 0.09239130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.020243240520358086, "learning_rate": 5e-05, "loss": 0.11132572591304779, "num_tokens": 2841741.0, "reward": -0.04345703125, "reward_std": 0.6613745093345642, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.4712120592594147, "rewards/length_penalty/mean": -0.7234570384025574, "rewards/length_penalty/std": 0.21751287579536438, "sampling/importance_sampling_ratio/max": 2.625032424926758, "sampling/importance_sampling_ratio/mean": 0.9916952252388, "sampling/importance_sampling_ratio/min": 0.20833513140678406, "sampling/sampling_logp_difference/max": 1.568607211112976, "sampling/sampling_logp_difference/mean": 0.01689102314412594, "step": 34, "step_time": 26.093267522752285 }, { "clip_ratio/high_max": 0.0007707345008384437, "clip_ratio/high_mean": 0.0007707345008384437, "clip_ratio/low_mean": 5.561825819313526e-05, "clip_ratio/low_min": 5.561825819313526e-05, "clip_ratio/region_mean": 0.0008263527532108128, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1967.0, "completions/mean_length": 1388.7999267578125, "completions/mean_terminated_length": 1157.189208984375, "completions/min_length": 717.0, "completions/min_terminated_length": 717.0, "entropy": 0.25725961923599244, "epoch": 0.09510869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.020236779004335403, "learning_rate": 5e-05, "loss": 0.10550195723772049, "num_tokens": 2913911.0, "reward": 0.06187499687075615, "reward_std": 0.646725058555603, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.6781250238418579, "rewards/length_penalty/std": 0.22613263130187988, "sampling/importance_sampling_ratio/max": 2.0787932872772217, "sampling/importance_sampling_ratio/mean": 0.990626335144043, "sampling/importance_sampling_ratio/min": 0.1848619133234024, "sampling/sampling_logp_difference/max": 1.6881461143493652, "sampling/sampling_logp_difference/mean": 0.018169380724430084, "step": 35, "step_time": 25.364839713089168 }, { "clip_ratio/high_max": 0.0006121197395259515, "clip_ratio/high_mean": 0.0006121197395259515, "clip_ratio/low_mean": 0.00011545439338078722, "clip_ratio/low_min": 0.00011545439338078722, "clip_ratio/region_mean": 0.0007275741430930793, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 2018.0, "completions/mean_length": 1189.8399658203125, "completions/mean_terminated_length": 1072.8182373046875, "completions/min_length": 626.0, "completions/min_terminated_length": 626.0, "entropy": 0.20621423423290253, "epoch": 0.09782608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.017101073637604713, "learning_rate": 5e-05, "loss": 0.08683709055185318, "num_tokens": 2975873.0, "reward": 0.2190234363079071, "reward_std": 0.609960675239563, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5809765458106995, "rewards/length_penalty/std": 0.22153306007385254, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.992645263671875, "sampling/importance_sampling_ratio/min": 0.14949937164783478, "sampling/sampling_logp_difference/max": 1.9004631042480469, "sampling/sampling_logp_difference/mean": 0.01638173498213291, "step": 36, "step_time": 25.151533517986536 }, { "clip_ratio/high_max": 0.0017051160102710128, "clip_ratio/high_mean": 0.0017051160102710128, "clip_ratio/low_mean": 4.560533707262948e-05, "clip_ratio/low_min": 4.560533707262948e-05, "clip_ratio/region_mean": 0.001750721363350749, "completions/clipped_ratio": 0.23999999463558197, "completions/max_length": 2048.0, "completions/max_terminated_length": 1696.0, "completions/mean_length": 1309.4599609375, "completions/mean_terminated_length": 1076.23681640625, "completions/min_length": 674.0, "completions/min_terminated_length": 674.0, "entropy": 0.2457641303539276, "epoch": 0.10054347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.01708856225013733, "learning_rate": 5e-05, "loss": 0.08991742134094238, "num_tokens": 3044866.0, "reward": -0.07938476651906967, "reward_std": 0.6378496289253235, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.6393847465515137, "rewards/length_penalty/std": 0.23324140906333923, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9911713600158691, "sampling/importance_sampling_ratio/min": 0.18068967759609222, "sampling/sampling_logp_difference/max": 1.7109742164611816, "sampling/sampling_logp_difference/mean": 0.01840140111744404, "step": 37, "step_time": 25.540588906034827 }, { "clip_ratio/high_max": 0.0007167195668444038, "clip_ratio/high_mean": 0.0007167195668444038, "clip_ratio/low_mean": 7.254186493810266e-05, "clip_ratio/low_min": 7.254186493810266e-05, "clip_ratio/region_mean": 0.0007892614346928894, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1988.0, "completions/mean_length": 1333.4000244140625, "completions/mean_terminated_length": 1235.95458984375, "completions/min_length": 690.0, "completions/min_terminated_length": 690.0, "entropy": 0.17886880338191985, "epoch": 0.10326086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.02032496966421604, "learning_rate": 5e-05, "loss": 0.08079551160335541, "num_tokens": 3114926.0, "reward": 0.16892577707767487, "reward_std": 0.5259281396865845, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.651074230670929, "rewards/length_penalty/std": 0.23055696487426758, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9937843680381775, "sampling/importance_sampling_ratio/min": 0.2627275884151459, "sampling/sampling_logp_difference/max": 1.3366376161575317, "sampling/sampling_logp_difference/mean": 0.013650714419782162, "step": 38, "step_time": 26.08059739181772 }, { "clip_ratio/high_max": 0.0005097337998449802, "clip_ratio/high_mean": 0.0005097337998449802, "clip_ratio/low_mean": 6.99934986187145e-05, "clip_ratio/low_min": 6.99934986187145e-05, "clip_ratio/region_mean": 0.0005797273013740778, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 1743.0, "completions/mean_length": 1207.47998046875, "completions/mean_terminated_length": 970.4102783203125, "completions/min_length": 488.0, "completions/min_terminated_length": 488.0, "entropy": 0.18856219053268433, "epoch": 0.10597826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.020054886117577553, "learning_rate": 5e-05, "loss": 0.0812826082110405, "num_tokens": 3177790.0, "reward": -0.009589843451976776, "reward_std": 0.7438336610794067, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.5895898342132568, "rewards/length_penalty/std": 0.26728230714797974, "sampling/importance_sampling_ratio/max": 2.8230764865875244, "sampling/importance_sampling_ratio/mean": 0.9934998750686646, "sampling/importance_sampling_ratio/min": 0.23803837597370148, "sampling/sampling_logp_difference/max": 1.4353233575820923, "sampling/sampling_logp_difference/mean": 0.01414400339126587, "step": 39, "step_time": 24.726267321035266 }, { "clip_ratio/high_max": 0.0008135152223985642, "clip_ratio/high_mean": 0.0008135152223985642, "clip_ratio/low_mean": 8.447097934549674e-05, "clip_ratio/low_min": 8.447097934549674e-05, "clip_ratio/region_mean": 0.0008979861973784864, "completions/clipped_ratio": 0.0, "completions/max_length": 1624.0, "completions/max_terminated_length": 1624.0, "completions/mean_length": 933.5399780273438, "completions/mean_terminated_length": 933.5399780273438, "completions/min_length": 641.0, "completions/min_terminated_length": 641.0, "entropy": 0.192103710770607, "epoch": 0.10869565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.014751175418496132, "learning_rate": 5e-05, "loss": 0.07916796952486038, "num_tokens": 3227187.0, "reward": 0.5441699028015137, "reward_std": 0.11200839281082153, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.45583006739616394, "rewards/length_penalty/std": 0.11200839281082153, "sampling/importance_sampling_ratio/max": 2.518812656402588, "sampling/importance_sampling_ratio/mean": 0.9928264021873474, "sampling/importance_sampling_ratio/min": 0.1412581503391266, "sampling/sampling_logp_difference/max": 1.9571661949157715, "sampling/sampling_logp_difference/mean": 0.016277069225907326, "step": 40, "step_time": 19.264488755259663 }, { "clip_ratio/high_max": 0.0006308750482276082, "clip_ratio/high_mean": 0.0006308750482276082, "clip_ratio/low_mean": 7.04108999343589e-05, "clip_ratio/low_min": 7.04108999343589e-05, "clip_ratio/region_mean": 0.0007012859452515841, "completions/clipped_ratio": 0.17999999225139618, "completions/max_length": 2048.0, "completions/max_terminated_length": 1934.0, "completions/mean_length": 1309.0999755859375, "completions/mean_terminated_length": 1146.90234375, "completions/min_length": 592.0, "completions/min_terminated_length": 592.0, "entropy": 0.21561425626277925, "epoch": 0.11141304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.020088374614715576, "learning_rate": 5e-05, "loss": 0.08004692196846008, "num_tokens": 3296382.0, "reward": -0.13920898735523224, "reward_std": 0.6492041945457458, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.639208972454071, "rewards/length_penalty/std": 0.229248046875, "sampling/importance_sampling_ratio/max": 2.644029140472412, "sampling/importance_sampling_ratio/mean": 0.9920929670333862, "sampling/importance_sampling_ratio/min": 0.07542523741722107, "sampling/sampling_logp_difference/max": 2.58461332321167, "sampling/sampling_logp_difference/mean": 0.016478460282087326, "step": 41, "step_time": 26.089902371168137 }, { "clip_ratio/high_max": 0.0007611454464495182, "clip_ratio/high_mean": 0.0007611454464495182, "clip_ratio/low_mean": 8.782477234490216e-05, "clip_ratio/low_min": 8.782477234490216e-05, "clip_ratio/region_mean": 0.0008489702187944204, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 1938.0, "completions/mean_length": 1232.4000244140625, "completions/mean_terminated_length": 1077.047607421875, "completions/min_length": 473.0, "completions/min_terminated_length": 473.0, "entropy": 0.19867940843105317, "epoch": 0.11413043478260869, "frac_reward_zero_std": 0.0, "grad_norm": 0.01986713334918022, "learning_rate": 5e-05, "loss": 0.06985890865325928, "num_tokens": 3360992.0, "reward": 0.23824217915534973, "reward_std": 0.5804083347320557, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.601757824420929, "rewards/length_penalty/std": 0.2639937698841095, "sampling/importance_sampling_ratio/max": 2.2874951362609863, "sampling/importance_sampling_ratio/mean": 0.9928309917449951, "sampling/importance_sampling_ratio/min": 0.11545022577047348, "sampling/sampling_logp_difference/max": 2.1589157581329346, "sampling/sampling_logp_difference/mean": 0.015996571630239487, "step": 42, "step_time": 24.868821669137105 }, { "clip_ratio/high_max": 0.000910443312022835, "clip_ratio/high_mean": 0.000910443312022835, "clip_ratio/low_mean": 6.609660922549666e-05, "clip_ratio/low_min": 6.609660922549666e-05, "clip_ratio/region_mean": 0.0009765399037860334, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1646.0, "completions/mean_length": 888.1199951171875, "completions/mean_terminated_length": 864.448974609375, "completions/min_length": 459.0, "completions/min_terminated_length": 459.0, "entropy": 0.21161480844020844, "epoch": 0.11684782608695653, "frac_reward_zero_std": 0.0, "grad_norm": 0.020094897598028183, "learning_rate": 5e-05, "loss": 0.06470935791730881, "num_tokens": 3407368.0, "reward": 0.48634764552116394, "reward_std": 0.3447928726673126, "rewards/correctness/mean": 0.9200000166893005, "rewards/correctness/std": 0.27404749393463135, "rewards/length_penalty/mean": -0.4336523413658142, "rewards/length_penalty/std": 0.17656175792217255, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9923710227012634, "sampling/importance_sampling_ratio/min": 0.11419941484928131, "sampling/sampling_logp_difference/max": 2.169809103012085, "sampling/sampling_logp_difference/mean": 0.018651597201824188, "step": 43, "step_time": 22.883227334124967 }, { "clip_ratio/high_max": 0.0008881673566065729, "clip_ratio/high_mean": 0.0008881673566065729, "clip_ratio/low_mean": 0.00012054646213073283, "clip_ratio/low_min": 0.00012054646213073283, "clip_ratio/region_mean": 0.001008713839109987, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 1955.0, "completions/mean_length": 1177.280029296875, "completions/mean_terminated_length": 1035.534912109375, "completions/min_length": 586.0, "completions/min_terminated_length": 586.0, "entropy": 0.2084593027830124, "epoch": 0.11956521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.019616451114416122, "learning_rate": 5e-05, "loss": 0.12185367941856384, "num_tokens": 3469232.0, "reward": 0.28515625, "reward_std": 0.5443253517150879, "rewards/correctness/mean": 0.8600000143051147, "rewards/correctness/std": 0.3505098223686218, "rewards/length_penalty/mean": -0.5748437643051147, "rewards/length_penalty/std": 0.22794675827026367, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9923920035362244, "sampling/importance_sampling_ratio/min": 0.1075812503695488, "sampling/sampling_logp_difference/max": 2.229508876800537, "sampling/sampling_logp_difference/mean": 0.0175347737967968, "step": 44, "step_time": 25.075815413612872 }, { "clip_ratio/high_max": 0.0009058942960109562, "clip_ratio/high_mean": 0.0009058942960109562, "clip_ratio/low_mean": 7.237063837237657e-05, "clip_ratio/low_min": 7.237063837237657e-05, "clip_ratio/region_mean": 0.0009782649285625666, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1153.760009765625, "completions/mean_terminated_length": 983.4285888671875, "completions/min_length": 451.0, "completions/min_terminated_length": 451.0, "entropy": 0.23205784261226653, "epoch": 0.12228260869565218, "frac_reward_zero_std": 0.0, "grad_norm": 0.021139081567525864, "learning_rate": 5e-05, "loss": 0.06741262972354889, "num_tokens": 3529860.0, "reward": 0.15664061903953552, "reward_std": 0.6854303479194641, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.5633593797683716, "rewards/length_penalty/std": 0.2703804671764374, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9912254214286804, "sampling/importance_sampling_ratio/min": 0.08344912528991699, "sampling/sampling_logp_difference/max": 2.483518123626709, "sampling/sampling_logp_difference/mean": 0.01880672574043274, "step": 45, "step_time": 24.49455920001492 }, { "clip_ratio/high_max": 0.0015542090637609362, "clip_ratio/high_mean": 0.0015542090637609362, "clip_ratio/low_mean": 8.50872034789063e-05, "clip_ratio/low_min": 8.50872034789063e-05, "clip_ratio/region_mean": 0.0016392962774261833, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 1179.5999755859375, "completions/mean_terminated_length": 962.5, "completions/min_length": 551.0, "completions/min_terminated_length": 551.0, "entropy": 0.23213451504707336, "epoch": 0.125, "frac_reward_zero_std": 0.0, "grad_norm": 0.016054287552833557, "learning_rate": 5e-05, "loss": 0.054812848567962646, "num_tokens": 3592350.0, "reward": 0.22402343153953552, "reward_std": 0.6261232495307922, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.575976550579071, "rewards/length_penalty/std": 0.23599623143672943, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.991656482219696, "sampling/importance_sampling_ratio/min": 0.0905810073018074, "sampling/sampling_logp_difference/max": 2.401510715484619, "sampling/sampling_logp_difference/mean": 0.018099654465913773, "step": 46, "step_time": 24.68149675289169 }, { "clip_ratio/high_max": 0.0008501063100993633, "clip_ratio/high_mean": 0.0008501063100993633, "clip_ratio/low_mean": 0.00010414896241854877, "clip_ratio/low_min": 0.00010414896241854877, "clip_ratio/region_mean": 0.0009542552754282952, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1882.0, "completions/mean_length": 944.0199584960938, "completions/mean_terminated_length": 898.0208740234375, "completions/min_length": 576.0, "completions/min_terminated_length": 576.0, "entropy": 0.19239770770072936, "epoch": 0.12771739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.020118683576583862, "learning_rate": 5e-05, "loss": 0.035656265914440155, "num_tokens": 3641941.0, "reward": 0.4390527307987213, "reward_std": 0.44222262501716614, "rewards/correctness/mean": 0.8999999761581421, "rewards/correctness/std": 0.30304577946662903, "rewards/length_penalty/mean": -0.46094727516174316, "rewards/length_penalty/std": 0.1884309947490692, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9925170540809631, "sampling/importance_sampling_ratio/min": 0.10729321092367172, "sampling/sampling_logp_difference/max": 2.232189893722534, "sampling/sampling_logp_difference/mean": 0.01707930490374565, "step": 47, "step_time": 23.979559221770614 }, { "clip_ratio/high_max": 0.0007360532443271949, "clip_ratio/high_mean": 0.0007360532443271949, "clip_ratio/low_mean": 5.149670469108969e-05, "clip_ratio/low_min": 5.149670469108969e-05, "clip_ratio/region_mean": 0.0007875499461079017, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 2012.0, "completions/mean_length": 1140.6199951171875, "completions/mean_terminated_length": 1039.800048828125, "completions/min_length": 514.0, "completions/min_terminated_length": 514.0, "entropy": 0.18729614913463594, "epoch": 0.13043478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.019802836701273918, "learning_rate": 5e-05, "loss": 0.06913310289382935, "num_tokens": 3701932.0, "reward": 0.12305663526058197, "reward_std": 0.6152191162109375, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.5569433569908142, "rewards/length_penalty/std": 0.24529792368412018, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9927173256874084, "sampling/importance_sampling_ratio/min": 0.13456226885318756, "sampling/sampling_logp_difference/max": 2.005728244781494, "sampling/sampling_logp_difference/mean": 0.016090555116534233, "step": 48, "step_time": 24.360856070183218 }, { "clip_ratio/high_max": 0.0014991959935287014, "clip_ratio/high_mean": 0.0014991959935287014, "clip_ratio/low_mean": 5.690939142368734e-05, "clip_ratio/low_min": 5.690939142368734e-05, "clip_ratio/region_mean": 0.001556105344207026, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1192.0, "completions/mean_length": 1090.3800048828125, "completions/mean_terminated_length": 850.9750366210938, "completions/min_length": 518.0, "completions/min_terminated_length": 518.0, "entropy": 0.23595961928367615, "epoch": 0.1331521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.015036417171359062, "learning_rate": 5e-05, "loss": 0.054190024733543396, "num_tokens": 3759141.0, "reward": 0.2675878703594208, "reward_std": 0.6470195651054382, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.5324121117591858, "rewards/length_penalty/std": 0.2540016770362854, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9912841320037842, "sampling/importance_sampling_ratio/min": 0.12618088722229004, "sampling/sampling_logp_difference/max": 2.0700387954711914, "sampling/sampling_logp_difference/mean": 0.019528958946466446, "step": 49, "step_time": 24.133017131127417 }, { "clip_ratio/high_max": 0.0015399554860778153, "clip_ratio/high_mean": 0.0015399554860778153, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015399554860778153, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 981.9400024414062, "completions/mean_terminated_length": 715.4249877929688, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.2847625881433487, "epoch": 0.1358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.01285274513065815, "learning_rate": 5e-05, "loss": 0.03275208920240402, "num_tokens": 3810648.0, "reward": 0.12053710967302322, "reward_std": 0.7175264954566956, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.4794628918170929, "rewards/length_penalty/std": 0.27745550870895386, "sampling/importance_sampling_ratio/max": 2.4426708221435547, "sampling/importance_sampling_ratio/mean": 0.9890289306640625, "sampling/importance_sampling_ratio/min": 0.10526978224515915, "sampling/sampling_logp_difference/max": 2.2512288093566895, "sampling/sampling_logp_difference/mean": 0.022413378581404686, "step": 50, "step_time": 24.217485108412802 }, { "clip_ratio/high_max": 0.0006860518013127148, "clip_ratio/high_mean": 0.0006860518013127148, "clip_ratio/low_mean": 6.418603006750346e-05, "clip_ratio/low_min": 6.418603006750346e-05, "clip_ratio/region_mean": 0.0007502378197386861, "completions/clipped_ratio": 0.0, "completions/max_length": 1852.0, "completions/max_terminated_length": 1852.0, "completions/mean_length": 936.3200073242188, "completions/mean_terminated_length": 936.3200073242188, "completions/min_length": 428.0, "completions/min_terminated_length": 428.0, "entropy": 0.19295797646045684, "epoch": 0.13858695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.016714785248041153, "learning_rate": 5e-05, "loss": 0.08077684044837952, "num_tokens": 3859974.0, "reward": 0.5428124666213989, "reward_std": 0.17791098356246948, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.4571875035762787, "rewards/length_penalty/std": 0.17791098356246948, "sampling/importance_sampling_ratio/max": 2.891409397125244, "sampling/importance_sampling_ratio/mean": 0.992777943611145, "sampling/importance_sampling_ratio/min": 0.12538689374923706, "sampling/sampling_logp_difference/max": 2.0763511657714844, "sampling/sampling_logp_difference/mean": 0.017098676413297653, "step": 51, "step_time": 21.442696853075176 }, { "clip_ratio/high_max": 0.0007797902682796121, "clip_ratio/high_mean": 0.0007797902682796121, "clip_ratio/low_mean": 4.9364475125912576e-05, "clip_ratio/low_min": 4.9364475125912576e-05, "clip_ratio/region_mean": 0.0008291547477710992, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 838.8800048828125, "completions/mean_terminated_length": 838.8800048828125, "completions/min_length": 474.0, "completions/min_terminated_length": 474.0, "entropy": 0.17070605158805846, "epoch": 0.14130434782608695, "frac_reward_zero_std": 0.0, "grad_norm": 0.015239793807268143, "learning_rate": 5e-05, "loss": 0.03420329466462135, "num_tokens": 3904178.0, "reward": 0.3903906047344208, "reward_std": 0.41191530227661133, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.4096093773841858, "rewards/length_penalty/std": 0.11314770579338074, "sampling/importance_sampling_ratio/max": 2.1938090324401855, "sampling/importance_sampling_ratio/mean": 0.9938638210296631, "sampling/importance_sampling_ratio/min": 0.08321992307901382, "sampling/sampling_logp_difference/max": 2.4862685203552246, "sampling/sampling_logp_difference/mean": 0.01698494143784046, "step": 52, "step_time": 16.543803959153593 }, { "clip_ratio/high_max": 0.0008488356252200902, "clip_ratio/high_mean": 0.0008488356252200902, "clip_ratio/low_mean": 4.159114178037271e-05, "clip_ratio/low_min": 4.159114178037271e-05, "clip_ratio/region_mean": 0.0008904267684556544, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 1766.0, "completions/mean_length": 1033.179931640625, "completions/mean_terminated_length": 944.934814453125, "completions/min_length": 453.0, "completions/min_terminated_length": 453.0, "entropy": 0.20474017262458802, "epoch": 0.14402173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.0204797200858593, "learning_rate": 5e-05, "loss": 0.05927962809801102, "num_tokens": 3958557.0, "reward": 0.19551756978034973, "reward_std": 0.6273199915885925, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.5044824481010437, "rewards/length_penalty/std": 0.22523540258407593, "sampling/importance_sampling_ratio/max": 1.9548008441925049, "sampling/importance_sampling_ratio/mean": 0.991944432258606, "sampling/importance_sampling_ratio/min": 0.04066736251115799, "sampling/sampling_logp_difference/max": 3.202329397201538, "sampling/sampling_logp_difference/mean": 0.0187305249273777, "step": 53, "step_time": 23.953327698865905 }, { "clip_ratio/high_max": 0.0005767365772044286, "clip_ratio/high_mean": 0.0005767365772044286, "clip_ratio/low_mean": 6.257046188693494e-05, "clip_ratio/low_min": 6.257046188693494e-05, "clip_ratio/region_mean": 0.0006393070274498313, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1871.0, "completions/mean_length": 1254.1199951171875, "completions/mean_terminated_length": 1055.6500244140625, "completions/min_length": 588.0, "completions/min_terminated_length": 588.0, "entropy": 0.20505343973636628, "epoch": 0.14673913043478262, "frac_reward_zero_std": 0.0, "grad_norm": 0.018724745139479637, "learning_rate": 5e-05, "loss": 0.07014630734920502, "num_tokens": 4025083.0, "reward": 0.14763671159744263, "reward_std": 0.6248001456260681, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.6123632788658142, "rewards/length_penalty/std": 0.24844162166118622, "sampling/importance_sampling_ratio/max": 2.6337807178497314, "sampling/importance_sampling_ratio/mean": 0.992132842540741, "sampling/importance_sampling_ratio/min": 0.11581636965274811, "sampling/sampling_logp_difference/max": 2.1557493209838867, "sampling/sampling_logp_difference/mean": 0.01712055131793022, "step": 54, "step_time": 25.70337758003734 }, { "clip_ratio/high_max": 0.0005223329411819577, "clip_ratio/high_mean": 0.0005223329411819577, "clip_ratio/low_mean": 0.00012916228006361053, "clip_ratio/low_min": 0.00012916228006361053, "clip_ratio/region_mean": 0.0006514952285215259, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 1897.0, "completions/mean_length": 1253.8399658203125, "completions/mean_terminated_length": 913.4857177734375, "completions/min_length": 545.0, "completions/min_terminated_length": 545.0, "entropy": 0.24083685576915742, "epoch": 0.14945652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.02363269031047821, "learning_rate": 5e-05, "loss": 0.03444070741534233, "num_tokens": 4090665.0, "reward": 0.08777343481779099, "reward_std": 0.7344273328781128, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.6122265458106995, "rewards/length_penalty/std": 0.303725928068161, "sampling/importance_sampling_ratio/max": 2.8206093311309814, "sampling/importance_sampling_ratio/mean": 0.9912818670272827, "sampling/importance_sampling_ratio/min": 0.14987419545650482, "sampling/sampling_logp_difference/max": 1.8979589939117432, "sampling/sampling_logp_difference/mean": 0.019693711772561073, "step": 55, "step_time": 25.123637669952586 }, { "clip_ratio/high_max": 0.0012042871094308794, "clip_ratio/high_mean": 0.0012042871094308794, "clip_ratio/low_mean": 0.00010765771730802953, "clip_ratio/low_min": 0.00010765771730802953, "clip_ratio/region_mean": 0.0013119448209181427, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1432.0, "completions/mean_length": 987.5599975585938, "completions/mean_terminated_length": 722.4500122070312, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "entropy": 0.1985310882329941, "epoch": 0.15217391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.014590896666049957, "learning_rate": 5e-05, "loss": 0.029594430699944496, "num_tokens": 4143113.0, "reward": 0.11779296398162842, "reward_std": 0.673450767993927, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.4822070300579071, "rewards/length_penalty/std": 0.29239600896835327, "sampling/importance_sampling_ratio/max": 2.725640296936035, "sampling/importance_sampling_ratio/mean": 0.9925079345703125, "sampling/importance_sampling_ratio/min": 0.15510861575603485, "sampling/sampling_logp_difference/max": 1.863629698753357, "sampling/sampling_logp_difference/mean": 0.017994750291109085, "step": 56, "step_time": 24.02179194591008 }, { "clip_ratio/high_max": 0.0005607875806163065, "clip_ratio/high_mean": 0.0005607875806163065, "clip_ratio/low_mean": 0.0001157178936409764, "clip_ratio/low_min": 0.0001157178936409764, "clip_ratio/region_mean": 0.0006765054655261338, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1959.0, "completions/mean_length": 1183.419921875, "completions/mean_terminated_length": 967.2750244140625, "completions/min_length": 448.0, "completions/min_terminated_length": 448.0, "entropy": 0.24734485745429993, "epoch": 0.15489130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.01673017628490925, "learning_rate": 5e-05, "loss": 0.04797259345650673, "num_tokens": 4205524.0, "reward": 0.22215819358825684, "reward_std": 0.6535060405731201, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5778418183326721, "rewards/length_penalty/std": 0.30248209834098816, "sampling/importance_sampling_ratio/max": 2.1797726154327393, "sampling/importance_sampling_ratio/mean": 0.9906326532363892, "sampling/importance_sampling_ratio/min": 0.17795737087726593, "sampling/sampling_logp_difference/max": 1.7262113094329834, "sampling/sampling_logp_difference/mean": 0.019809162244200706, "step": 57, "step_time": 25.37448728713207 }, { "clip_ratio/high_max": 0.0006756895920261741, "clip_ratio/high_mean": 0.0006756895920261741, "clip_ratio/low_mean": 0.0001676665517152287, "clip_ratio/low_min": 0.0001676665517152287, "clip_ratio/region_mean": 0.0008433561422862112, "completions/clipped_ratio": 0.23999999463558197, "completions/max_length": 2048.0, "completions/max_terminated_length": 1995.0, "completions/mean_length": 1314.719970703125, "completions/mean_terminated_length": 1083.157958984375, "completions/min_length": 486.0, "completions/min_terminated_length": 486.0, "entropy": 0.24679793417453766, "epoch": 0.15760869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.023087836802005768, "learning_rate": 5e-05, "loss": 0.07203301787376404, "num_tokens": 4275070.0, "reward": -0.001953125, "reward_std": 0.7374632358551025, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732054233551, "rewards/length_penalty/mean": -0.6419531106948853, "rewards/length_penalty/std": 0.2896905243396759, "sampling/importance_sampling_ratio/max": 2.4132602214813232, "sampling/importance_sampling_ratio/mean": 0.9908359050750732, "sampling/importance_sampling_ratio/min": 0.09266611188650131, "sampling/sampling_logp_difference/max": 2.3787524700164795, "sampling/sampling_logp_difference/mean": 0.020349619910120964, "step": 58, "step_time": 25.843243841081858 }, { "clip_ratio/high_max": 0.0007752020144835114, "clip_ratio/high_mean": 0.0007752020144835114, "clip_ratio/low_mean": 0.0001578759402036667, "clip_ratio/low_min": 0.0001578759402036667, "clip_ratio/region_mean": 0.000933077948866412, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 1734.0, "completions/mean_length": 909.1599731445312, "completions/mean_terminated_length": 810.1304321289062, "completions/min_length": 438.0, "completions/min_terminated_length": 438.0, "entropy": 0.1871820569038391, "epoch": 0.16032608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.021188728511333466, "learning_rate": 5e-05, "loss": 0.07215945422649384, "num_tokens": 4323238.0, "reward": 0.4360741972923279, "reward_std": 0.5008804798126221, "rewards/correctness/mean": 0.8799999952316284, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.44392576813697815, "rewards/length_penalty/std": 0.22381433844566345, "sampling/importance_sampling_ratio/max": 2.9011504650115967, "sampling/importance_sampling_ratio/mean": 0.9930024743080139, "sampling/importance_sampling_ratio/min": 0.1250293254852295, "sampling/sampling_logp_difference/max": 2.079206943511963, "sampling/sampling_logp_difference/mean": 0.018969286233186722, "step": 59, "step_time": 23.926262149121612 }, { "clip_ratio/high_max": 0.0004012422810774297, "clip_ratio/high_mean": 0.0004012422810774297, "clip_ratio/low_mean": 4.353504627943039e-05, "clip_ratio/low_min": 4.353504627943039e-05, "clip_ratio/region_mean": 0.00044477732735686003, "completions/clipped_ratio": 0.0, "completions/max_length": 1467.0, "completions/max_terminated_length": 1467.0, "completions/mean_length": 893.9400024414062, "completions/mean_terminated_length": 893.9400024414062, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.15323590338230134, "epoch": 0.16304347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.013462953269481659, "learning_rate": 5e-05, "loss": 0.026647938415408134, "num_tokens": 4370745.0, "reward": -0.05649413913488388, "reward_std": 0.5149266123771667, "rewards/correctness/mean": 0.3799999952316284, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.4364941418170929, "rewards/length_penalty/std": 0.13239894807338715, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9946074485778809, "sampling/importance_sampling_ratio/min": 0.07585878670215607, "sampling/sampling_logp_difference/max": 2.5788817405700684, "sampling/sampling_logp_difference/mean": 0.015621195547282696, "step": 60, "step_time": 17.78750504530035 }, { "clip_ratio/high_max": 0.0007783836219459773, "clip_ratio/high_mean": 0.0007783836219459773, "clip_ratio/low_mean": 7.167461153585463e-05, "clip_ratio/low_min": 7.167461153585463e-05, "clip_ratio/region_mean": 0.0008500582538545131, "completions/clipped_ratio": 0.0, "completions/max_length": 1332.0, "completions/max_terminated_length": 1332.0, "completions/mean_length": 816.8800048828125, "completions/mean_terminated_length": 816.8800048828125, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.16096674501895905, "epoch": 0.16576086956521738, "frac_reward_zero_std": 0.0, "grad_norm": 0.01674392633140087, "learning_rate": 5e-05, "loss": 0.04878897964954376, "num_tokens": 4413999.0, "reward": 0.6011328101158142, "reward_std": 0.08173488825559616, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.3988671898841858, "rewards/length_penalty/std": 0.08173488825559616, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9938045740127563, "sampling/importance_sampling_ratio/min": 0.03371766582131386, "sampling/sampling_logp_difference/max": 3.38973331451416, "sampling/sampling_logp_difference/mean": 0.01792747713625431, "step": 61, "step_time": 15.884311718866229 }, { "clip_ratio/high_max": 0.0005490639596246183, "clip_ratio/high_mean": 0.0005490639596246183, "clip_ratio/low_mean": 8.991758077172562e-05, "clip_ratio/low_min": 8.991758077172562e-05, "clip_ratio/region_mean": 0.0006389815418515355, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 1749.0, "completions/mean_length": 1040.5399169921875, "completions/mean_terminated_length": 976.2340087890625, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.15563784837722777, "epoch": 0.16847826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.018042365089058876, "learning_rate": 5e-05, "loss": 0.07988356798887253, "num_tokens": 4469646.0, "reward": 0.33192381262779236, "reward_std": 0.5288719534873962, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.5080761909484863, "rewards/length_penalty/std": 0.21403193473815918, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9942488074302673, "sampling/importance_sampling_ratio/min": 0.06006976589560509, "sampling/sampling_logp_difference/max": 2.812248706817627, "sampling/sampling_logp_difference/mean": 0.015519483014941216, "step": 62, "step_time": 24.003339575137943 }, { "clip_ratio/high_max": 0.0007015687064267695, "clip_ratio/high_mean": 0.0007015687064267695, "clip_ratio/low_mean": 0.00010667735186871141, "clip_ratio/low_min": 0.00010667735186871141, "clip_ratio/region_mean": 0.0008082460495643318, "completions/clipped_ratio": 0.0, "completions/max_length": 1687.0, "completions/max_terminated_length": 1687.0, "completions/mean_length": 828.7799682617188, "completions/mean_terminated_length": 828.7799682617188, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.16558827161788942, "epoch": 0.17119565217391305, "frac_reward_zero_std": 0.0, "grad_norm": 0.016367904841899872, "learning_rate": 5e-05, "loss": 0.04421332851052284, "num_tokens": 4514335.0, "reward": 0.3953222632408142, "reward_std": 0.5528039932250977, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.40467774868011475, "rewards/length_penalty/std": 0.1717558354139328, "sampling/importance_sampling_ratio/max": 2.7936859130859375, "sampling/importance_sampling_ratio/mean": 0.9938439130783081, "sampling/importance_sampling_ratio/min": 0.05281807854771614, "sampling/sampling_logp_difference/max": 2.940901756286621, "sampling/sampling_logp_difference/mean": 0.017519377171993256, "step": 63, "step_time": 20.010241974843666 }, { "clip_ratio/high_max": 0.0006772913970053196, "clip_ratio/high_mean": 0.0006772913970053196, "clip_ratio/low_mean": 6.347343005472795e-05, "clip_ratio/low_min": 6.347343005472795e-05, "clip_ratio/region_mean": 0.0007407648256048561, "completions/clipped_ratio": 0.0, "completions/max_length": 2002.0, "completions/max_terminated_length": 2002.0, "completions/mean_length": 881.5399780273438, "completions/mean_terminated_length": 881.5399780273438, "completions/min_length": 503.0, "completions/min_terminated_length": 503.0, "entropy": 0.16818468570709227, "epoch": 0.17391304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.016808470711112022, "learning_rate": 5e-05, "loss": 0.010990206152200699, "num_tokens": 4561932.0, "reward": 0.5095605254173279, "reward_std": 0.27923068404197693, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979365825653, "rewards/length_penalty/mean": -0.43043944239616394, "rewards/length_penalty/std": 0.12210877984762192, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9936347007751465, "sampling/importance_sampling_ratio/min": 0.11113839596509933, "sampling/sampling_logp_difference/max": 2.19697904586792, "sampling/sampling_logp_difference/mean": 0.018134895712137222, "step": 64, "step_time": 22.430961188860238 }, { "clip_ratio/high_max": 0.0006107009830884635, "clip_ratio/high_mean": 0.0006107009830884635, "clip_ratio/low_mean": 0.00012073849065927789, "clip_ratio/low_min": 0.00012073849065927789, "clip_ratio/region_mean": 0.000731439475202933, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 1452.0, "completions/mean_length": 999.5, "completions/mean_terminated_length": 828.81396484375, "completions/min_length": 454.0, "completions/min_terminated_length": 454.0, "entropy": 0.2120996743440628, "epoch": 0.1766304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.01858065277338028, "learning_rate": 5e-05, "loss": 0.07004983723163605, "num_tokens": 4615617.0, "reward": 0.3319628834724426, "reward_std": 0.5958665013313293, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.488037109375, "rewards/length_penalty/std": 0.233457550406456, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9916940331459045, "sampling/importance_sampling_ratio/min": 0.0842479020357132, "sampling/sampling_logp_difference/max": 2.473991632461548, "sampling/sampling_logp_difference/mean": 0.020551443099975586, "step": 65, "step_time": 23.88459292682819 }, { "clip_ratio/high_max": 0.0006235995824681595, "clip_ratio/high_mean": 0.0006235995824681595, "clip_ratio/low_mean": 0.00022068115940783174, "clip_ratio/low_min": 0.00022068115940783174, "clip_ratio/region_mean": 0.0008442807185929269, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 1759.0, "completions/mean_length": 875.4400024414062, "completions/mean_terminated_length": 684.5581665039062, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.2801995426416397, "epoch": 0.1793478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.01836577244102955, "learning_rate": 5e-05, "loss": 0.0533025860786438, "num_tokens": 4662439.0, "reward": 0.37253904342651367, "reward_std": 0.6679306626319885, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.4274609386920929, "rewards/length_penalty/std": 0.2749612033367157, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9891699552536011, "sampling/importance_sampling_ratio/min": 0.09494689106941223, "sampling/sampling_logp_difference/max": 2.3544375896453857, "sampling/sampling_logp_difference/mean": 0.025045860558748245, "step": 66, "step_time": 24.01824735174887 }, { "clip_ratio/high_max": 0.0006467935570981353, "clip_ratio/high_mean": 0.0006467935570981353, "clip_ratio/low_mean": 4.287245392333716e-05, "clip_ratio/low_min": 4.287245392333716e-05, "clip_ratio/region_mean": 0.0006896660139318556, "completions/clipped_ratio": 0.0, "completions/max_length": 1456.0, "completions/max_terminated_length": 1456.0, "completions/mean_length": 827.5399780273438, "completions/mean_terminated_length": 827.5399780273438, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.1334616631269455, "epoch": 0.18206521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.015385694801807404, "learning_rate": 5e-05, "loss": 0.042910344898700714, "num_tokens": 4706766.0, "reward": 0.1959277242422104, "reward_std": 0.5535297393798828, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.40407225489616394, "rewards/length_penalty/std": 0.1448303610086441, "sampling/importance_sampling_ratio/max": 2.695427417755127, "sampling/importance_sampling_ratio/mean": 0.9948953986167908, "sampling/importance_sampling_ratio/min": 0.09551125019788742, "sampling/sampling_logp_difference/max": 2.348511219024658, "sampling/sampling_logp_difference/mean": 0.01522344071418047, "step": 67, "step_time": 16.992219537962228 }, { "clip_ratio/high_max": 0.0005427057622000575, "clip_ratio/high_mean": 0.0005427057622000575, "clip_ratio/low_mean": 6.446797924581915e-05, "clip_ratio/low_min": 6.446797924581915e-05, "clip_ratio/region_mean": 0.0006071737268939614, "completions/clipped_ratio": 0.0, "completions/max_length": 1989.0, "completions/max_terminated_length": 1989.0, "completions/mean_length": 871.8800048828125, "completions/mean_terminated_length": 871.8800048828125, "completions/min_length": 508.0, "completions/min_terminated_length": 508.0, "entropy": 0.1475384384393692, "epoch": 0.18478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.017730075865983963, "learning_rate": 5e-05, "loss": 0.05759969353675842, "num_tokens": 4754120.0, "reward": 0.3742773234844208, "reward_std": 0.5240140557289124, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.4257226586341858, "rewards/length_penalty/std": 0.14220459759235382, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9945562481880188, "sampling/importance_sampling_ratio/min": 0.09534671157598495, "sampling/sampling_logp_difference/max": 2.3502354621887207, "sampling/sampling_logp_difference/mean": 0.01613455079495907, "step": 68, "step_time": 22.368666774127632 }, { "clip_ratio/high_max": 0.0006822091600042768, "clip_ratio/high_mean": 0.0006822091600042768, "clip_ratio/low_mean": 0.00017659609293332323, "clip_ratio/low_min": 0.00017659609293332323, "clip_ratio/region_mean": 0.0008588052645791322, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 1823.0, "completions/mean_length": 852.5, "completions/mean_terminated_length": 624.7857055664062, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "entropy": 0.22420361936092376, "epoch": 0.1875, "frac_reward_zero_std": 0.0, "grad_norm": 0.01747017167508602, "learning_rate": 5e-05, "loss": 0.045437250286340714, "num_tokens": 4799125.0, "reward": 0.18374022841453552, "reward_std": 0.6997568011283875, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.416259765625, "rewards/length_penalty/std": 0.2845315635204315, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.99150550365448, "sampling/importance_sampling_ratio/min": 0.08363517373800278, "sampling/sampling_logp_difference/max": 2.4812910556793213, "sampling/sampling_logp_difference/mean": 0.02138812094926834, "step": 69, "step_time": 23.31697426387109 }, { "clip_ratio/high_max": 0.0008050206408370287, "clip_ratio/high_mean": 0.0008050206408370287, "clip_ratio/low_mean": 5.3399032913148405e-05, "clip_ratio/low_min": 5.3399032913148405e-05, "clip_ratio/region_mean": 0.000858419673750177, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1751.0, "completions/mean_length": 1082.97998046875, "completions/mean_terminated_length": 841.7250366210938, "completions/min_length": 435.0, "completions/min_terminated_length": 435.0, "entropy": 0.14042216837406157, "epoch": 0.19021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.01626087725162506, "learning_rate": 5e-05, "loss": 0.04337764531373978, "num_tokens": 4856784.0, "reward": -0.3287988305091858, "reward_std": 0.583001434803009, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5287988185882568, "rewards/length_penalty/std": 0.2958765923976898, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9947971701622009, "sampling/importance_sampling_ratio/min": 0.1584167629480362, "sampling/sampling_logp_difference/max": 1.8425259590148926, "sampling/sampling_logp_difference/mean": 0.014615754596889019, "step": 70, "step_time": 24.817883169045672 }, { "clip_ratio/high_max": 0.0006307939882390201, "clip_ratio/high_mean": 0.0006307939882390201, "clip_ratio/low_mean": 0.00010670171468518674, "clip_ratio/low_min": 0.00010670171468518674, "clip_ratio/region_mean": 0.0007374957029242068, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1123.0, "completions/mean_length": 677.9400024414062, "completions/mean_terminated_length": 620.8541870117188, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.17736737430095673, "epoch": 0.19293478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.015899593010544777, "learning_rate": 5e-05, "loss": 0.04084380343556404, "num_tokens": 4893041.0, "reward": 0.20897459983825684, "reward_std": 0.5713562965393066, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.3310253918170929, "rewards/length_penalty/std": 0.16477957367897034, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9929877519607544, "sampling/importance_sampling_ratio/min": 0.0540241040289402, "sampling/sampling_logp_difference/max": 2.9183249473571777, "sampling/sampling_logp_difference/mean": 0.02107720635831356, "step": 71, "step_time": 22.17649055412039 }, { "clip_ratio/high_max": 0.0007707292737904936, "clip_ratio/high_mean": 0.0007707292737904936, "clip_ratio/low_mean": 9.523197077214718e-05, "clip_ratio/low_min": 9.523197077214718e-05, "clip_ratio/region_mean": 0.0008659612271003425, "completions/clipped_ratio": 0.0, "completions/max_length": 1150.0, "completions/max_terminated_length": 1150.0, "completions/mean_length": 638.5599975585938, "completions/mean_terminated_length": 638.5599975585938, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.16127082109451293, "epoch": 0.1956521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.015706723555922508, "learning_rate": 5e-05, "loss": 0.012553978711366653, "num_tokens": 4928639.0, "reward": 0.42820310592651367, "reward_std": 0.5053699612617493, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.3117968738079071, "rewards/length_penalty/std": 0.0972241684794426, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9939435124397278, "sampling/importance_sampling_ratio/min": 0.07872314006090164, "sampling/sampling_logp_difference/max": 2.541818141937256, "sampling/sampling_logp_difference/mean": 0.020169558003544807, "step": 72, "step_time": 13.558618172770366 }, { "clip_ratio/high_max": 0.0005829828616697342, "clip_ratio/high_mean": 0.0005829828616697342, "clip_ratio/low_mean": 0.00014270264655351638, "clip_ratio/low_min": 0.00014270264655351638, "clip_ratio/region_mean": 0.0007256855198647827, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 1980.0, "completions/mean_length": 888.760009765625, "completions/mean_terminated_length": 787.95654296875, "completions/min_length": 440.0, "completions/min_terminated_length": 440.0, "entropy": 0.19430480599403382, "epoch": 0.1983695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.023028841242194176, "learning_rate": 5e-05, "loss": 0.03241705894470215, "num_tokens": 4975487.0, "reward": 0.22603514790534973, "reward_std": 0.6428487300872803, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.47851812839508057, "rewards/length_penalty/mean": -0.4339648485183716, "rewards/length_penalty/std": 0.2522260248661041, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.992754340171814, "sampling/importance_sampling_ratio/min": 0.1667802333831787, "sampling/sampling_logp_difference/max": 1.7910783290863037, "sampling/sampling_logp_difference/mean": 0.019066043198108673, "step": 73, "step_time": 23.85767387226224 }, { "clip_ratio/high_max": 0.0005934039130806923, "clip_ratio/high_mean": 0.0005934039130806923, "clip_ratio/low_mean": 0.00011007156572304667, "clip_ratio/low_min": 0.00011007156572304667, "clip_ratio/region_mean": 0.0007034754729829728, "completions/clipped_ratio": 0.0, "completions/max_length": 1745.0, "completions/max_terminated_length": 1745.0, "completions/mean_length": 641.0799560546875, "completions/mean_terminated_length": 641.0799560546875, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.1928534835577011, "epoch": 0.20108695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.019313158467411995, "learning_rate": 5e-05, "loss": 0.05762927606701851, "num_tokens": 5010871.0, "reward": 0.08697265386581421, "reward_std": 0.5515088438987732, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.31302735209465027, "rewards/length_penalty/std": 0.12540066242218018, "sampling/importance_sampling_ratio/max": 2.433044910430908, "sampling/importance_sampling_ratio/mean": 0.9928313493728638, "sampling/importance_sampling_ratio/min": 0.08987344801425934, "sampling/sampling_logp_difference/max": 2.4093527793884277, "sampling/sampling_logp_difference/mean": 0.021775977686047554, "step": 74, "step_time": 19.006243550917134 }, { "clip_ratio/high_max": 0.000405387906357646, "clip_ratio/high_mean": 0.000405387906357646, "clip_ratio/low_mean": 0.00012361486005829647, "clip_ratio/low_min": 0.00012361486005829647, "clip_ratio/region_mean": 0.0005290027736919, "completions/clipped_ratio": 0.0, "completions/max_length": 1529.0, "completions/max_terminated_length": 1529.0, "completions/mean_length": 633.239990234375, "completions/mean_terminated_length": 633.239990234375, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.17757215201854706, "epoch": 0.20380434782608695, "frac_reward_zero_std": 0.0, "grad_norm": 0.020598672330379486, "learning_rate": 5e-05, "loss": 0.03665365278720856, "num_tokens": 5044863.0, "reward": 0.49080076813697815, "reward_std": 0.45810505747795105, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.3091992139816284, "rewards/length_penalty/std": 0.1596784144639969, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9929429292678833, "sampling/importance_sampling_ratio/min": 0.06892305612564087, "sampling/sampling_logp_difference/max": 2.674764633178711, "sampling/sampling_logp_difference/mean": 0.021487612277269363, "step": 75, "step_time": 16.96747936704196 }, { "clip_ratio/high_max": 0.0008531624858733267, "clip_ratio/high_mean": 0.0008531624858733267, "clip_ratio/low_mean": 1.858218020061031e-05, "clip_ratio/low_min": 1.858218020061031e-05, "clip_ratio/region_mean": 0.0008717446587979793, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 658.0599975585938, "completions/mean_terminated_length": 503.6222229003906, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "entropy": 0.24603551626205444, "epoch": 0.20652173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.012405374087393284, "learning_rate": 5e-05, "loss": 0.025707000866532326, "num_tokens": 5081106.0, "reward": 0.43868163228034973, "reward_std": 0.6422238349914551, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.3213183581829071, "rewards/length_penalty/std": 0.255402535200119, "sampling/importance_sampling_ratio/max": 2.560499906539917, "sampling/importance_sampling_ratio/mean": 0.9905421733856201, "sampling/importance_sampling_ratio/min": 0.1971578299999237, "sampling/sampling_logp_difference/max": 1.6237506866455078, "sampling/sampling_logp_difference/mean": 0.0244895052164793, "step": 76, "step_time": 22.57146995374933 }, { "clip_ratio/high_max": 0.0003801504441071302, "clip_ratio/high_mean": 0.0003801504441071302, "clip_ratio/low_mean": 0.00011223704786971212, "clip_ratio/low_min": 0.00011223704786971212, "clip_ratio/region_mean": 0.0004923874977976084, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1808.0, "completions/mean_length": 628.7799682617188, "completions/mean_terminated_length": 569.6458740234375, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.20442797541618346, "epoch": 0.20923913043478262, "frac_reward_zero_std": 0.0, "grad_norm": 0.019539417698979378, "learning_rate": 5e-05, "loss": 0.04475327581167221, "num_tokens": 5115105.0, "reward": 0.552978515625, "reward_std": 0.547501802444458, "rewards/correctness/mean": 0.8600000143051147, "rewards/correctness/std": 0.3505098223686218, "rewards/length_penalty/mean": -0.30702149868011475, "rewards/length_penalty/std": 0.22665424644947052, "sampling/importance_sampling_ratio/max": 2.7733845710754395, "sampling/importance_sampling_ratio/mean": 0.992488443851471, "sampling/importance_sampling_ratio/min": 0.21041618287563324, "sampling/sampling_logp_difference/max": 1.5586678981781006, "sampling/sampling_logp_difference/mean": 0.021170448511838913, "step": 77, "step_time": 22.34465615078807 }, { "clip_ratio/high_max": 0.0006306407682131976, "clip_ratio/high_mean": 0.0006306407682131976, "clip_ratio/low_mean": 3.474635304883123e-05, "clip_ratio/low_min": 3.474635304883123e-05, "clip_ratio/region_mean": 0.0006653871096204966, "completions/clipped_ratio": 0.0, "completions/max_length": 1658.0, "completions/max_terminated_length": 1658.0, "completions/mean_length": 599.6599731445312, "completions/mean_terminated_length": 599.6599731445312, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.1244092807173729, "epoch": 0.21195652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.015446798875927925, "learning_rate": 5e-05, "loss": 0.033272288739681244, "num_tokens": 5147388.0, "reward": 0.26719725131988525, "reward_std": 0.5827784538269043, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.29280272126197815, "rewards/length_penalty/std": 0.13731949031352997, "sampling/importance_sampling_ratio/max": 2.4936909675598145, "sampling/importance_sampling_ratio/mean": 0.994698166847229, "sampling/importance_sampling_ratio/min": 0.31123581528663635, "sampling/sampling_logp_difference/max": 1.1672043800354004, "sampling/sampling_logp_difference/mean": 0.0171463992446661, "step": 78, "step_time": 18.60519856400788 }, { "clip_ratio/high_max": 0.0008912550430977717, "clip_ratio/high_mean": 0.0008912550430977717, "clip_ratio/low_mean": 6.107631779741495e-05, "clip_ratio/low_min": 6.107631779741495e-05, "clip_ratio/region_mean": 0.0009523313667159528, "completions/clipped_ratio": 0.0, "completions/max_length": 1229.0, "completions/max_terminated_length": 1229.0, "completions/mean_length": 649.219970703125, "completions/mean_terminated_length": 649.219970703125, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.13126298934221267, "epoch": 0.21467391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.0188016165047884, "learning_rate": 5e-05, "loss": 0.015198985114693642, "num_tokens": 5183309.0, "reward": 0.4429980516433716, "reward_std": 0.4340318739414215, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.31700193881988525, "rewards/length_penalty/std": 0.08376923203468323, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9953542351722717, "sampling/importance_sampling_ratio/min": 0.24312418699264526, "sampling/sampling_logp_difference/max": 1.5119855403900146, "sampling/sampling_logp_difference/mean": 0.018735092133283615, "step": 79, "step_time": 14.177364142145962 }, { "clip_ratio/high_max": 0.0004742158984299749, "clip_ratio/high_mean": 0.0004742158984299749, "clip_ratio/low_mean": 0.0001656829408602789, "clip_ratio/low_min": 0.0001656829408602789, "clip_ratio/region_mean": 0.0006398988363798708, "completions/clipped_ratio": 0.0, "completions/max_length": 1211.0, "completions/max_terminated_length": 1211.0, "completions/mean_length": 603.4199829101562, "completions/mean_terminated_length": 603.4199829101562, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.15212061405181884, "epoch": 0.21739130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.0165855772793293, "learning_rate": 5e-05, "loss": 0.041541777551174164, "num_tokens": 5216640.0, "reward": 0.10536132752895355, "reward_std": 0.5482556223869324, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.29463866353034973, "rewards/length_penalty/std": 0.07611750066280365, "sampling/importance_sampling_ratio/max": 2.329467535018921, "sampling/importance_sampling_ratio/mean": 0.9940717816352844, "sampling/importance_sampling_ratio/min": 0.20993992686271667, "sampling/sampling_logp_difference/max": 1.5609338283538818, "sampling/sampling_logp_difference/mean": 0.019620422273874283, "step": 80, "step_time": 13.800314102787524 }, { "clip_ratio/high_max": 0.000613589191925712, "clip_ratio/high_mean": 0.000613589191925712, "clip_ratio/low_mean": 8.642562024760992e-05, "clip_ratio/low_min": 8.642562024760992e-05, "clip_ratio/region_mean": 0.0007000148179940879, "completions/clipped_ratio": 0.0, "completions/max_length": 1088.0, "completions/max_terminated_length": 1088.0, "completions/mean_length": 649.4199829101562, "completions/mean_terminated_length": 649.4199829101562, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.15520131587982178, "epoch": 0.22010869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.020023388788104057, "learning_rate": 5e-05, "loss": 0.010980712249875069, "num_tokens": 5253571.0, "reward": 0.42290037870407104, "reward_std": 0.3821259140968323, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308748841285706, "rewards/length_penalty/mean": -0.31709960103034973, "rewards/length_penalty/std": 0.09378077834844589, "sampling/importance_sampling_ratio/max": 2.707496404647827, "sampling/importance_sampling_ratio/mean": 0.9943109154701233, "sampling/importance_sampling_ratio/min": 0.11866816133260727, "sampling/sampling_logp_difference/max": 2.1314241886138916, "sampling/sampling_logp_difference/mean": 0.0196909848600626, "step": 81, "step_time": 13.374773803167045 }, { "clip_ratio/high_max": 0.0003143954527331516, "clip_ratio/high_mean": 0.0003143954527331516, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0003143954527331516, "completions/clipped_ratio": 0.0, "completions/max_length": 748.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 444.5799865722656, "completions/mean_terminated_length": 444.5799865722656, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.1351422816514969, "epoch": 0.22282608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.013173939660191536, "learning_rate": 5e-05, "loss": 0.020658979192376137, "num_tokens": 5278650.0, "reward": 0.5829198956489563, "reward_std": 0.46361181139945984, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.21708007156848907, "rewards/length_penalty/std": 0.06732401251792908, "sampling/importance_sampling_ratio/max": 2.7757651805877686, "sampling/importance_sampling_ratio/mean": 0.9944673776626587, "sampling/importance_sampling_ratio/min": 0.31246867775917053, "sampling/sampling_logp_difference/max": 1.1632510423660278, "sampling/sampling_logp_difference/mean": 0.021512571722269058, "step": 82, "step_time": 9.464399790856987 }, { "clip_ratio/high_max": 0.0006479248695541173, "clip_ratio/high_mean": 0.0006479248695541173, "clip_ratio/low_mean": 0.00010048146359622478, "clip_ratio/low_min": 0.00010048146359622478, "clip_ratio/region_mean": 0.0007484063331503421, "completions/clipped_ratio": 0.0, "completions/max_length": 720.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 408.0799865722656, "completions/mean_terminated_length": 408.0799865722656, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.1544148862361908, "epoch": 0.22554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.014162655919790268, "learning_rate": 5e-05, "loss": 0.009620768949389458, "num_tokens": 5301484.0, "reward": 0.3607421815395355, "reward_std": 0.4975096583366394, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.19925780594348907, "rewards/length_penalty/std": 0.06029851362109184, "sampling/importance_sampling_ratio/max": 2.475653648376465, "sampling/importance_sampling_ratio/mean": 0.9939197301864624, "sampling/importance_sampling_ratio/min": 0.22558780014514923, "sampling/sampling_logp_difference/max": 1.4890458583831787, "sampling/sampling_logp_difference/mean": 0.023020122200250626, "step": 83, "step_time": 8.50258247810416 }, { "clip_ratio/high_max": 0.0009030202636495233, "clip_ratio/high_mean": 0.0009030202636495233, "clip_ratio/low_mean": 7.876795716583729e-05, "clip_ratio/low_min": 7.876795716583729e-05, "clip_ratio/region_mean": 0.0009817882324568926, "completions/clipped_ratio": 0.0, "completions/max_length": 1560.0, "completions/max_terminated_length": 1560.0, "completions/mean_length": 530.8399658203125, "completions/mean_terminated_length": 530.8399658203125, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.14245042204856873, "epoch": 0.22826086956521738, "frac_reward_zero_std": 0.0, "grad_norm": 0.014098316431045532, "learning_rate": 5e-05, "loss": 0.028290294110774994, "num_tokens": 5331006.0, "reward": 0.5408007502555847, "reward_std": 0.3885652720928192, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.25919923186302185, "rewards/length_penalty/std": 0.1674625128507614, "sampling/importance_sampling_ratio/max": 2.278921604156494, "sampling/importance_sampling_ratio/mean": 0.9943795800209045, "sampling/importance_sampling_ratio/min": 0.21966639161109924, "sampling/sampling_logp_difference/max": 1.5156453847885132, "sampling/sampling_logp_difference/mean": 0.019804934039711952, "step": 84, "step_time": 17.037076706066728 }, { "clip_ratio/high_max": 0.0018277494236826897, "clip_ratio/high_mean": 0.0018277494236826897, "clip_ratio/low_mean": 0.00013214687642175703, "clip_ratio/low_min": 0.00013214687642175703, "clip_ratio/region_mean": 0.001959896294283681, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 2027.0, "completions/mean_length": 700.8399658203125, "completions/mean_terminated_length": 644.7083740234375, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.2856750160455704, "epoch": 0.23097826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.026974737644195557, "learning_rate": 5e-05, "loss": 0.09304343163967133, "num_tokens": 5368068.0, "reward": 0.3377929627895355, "reward_std": 0.5755784511566162, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.34220704436302185, "rewards/length_penalty/std": 0.2737504541873932, "sampling/importance_sampling_ratio/max": 2.973176956176758, "sampling/importance_sampling_ratio/mean": 0.988871157169342, "sampling/importance_sampling_ratio/min": 0.13172246515750885, "sampling/sampling_logp_difference/max": 2.0270581245422363, "sampling/sampling_logp_difference/mean": 0.028557635843753815, "step": 85, "step_time": 22.43926484696567 }, { "clip_ratio/high_max": 0.001044186291983351, "clip_ratio/high_mean": 0.001044186291983351, "clip_ratio/low_mean": 0.00011646859929896891, "clip_ratio/low_min": 0.00011646859929896891, "clip_ratio/region_mean": 0.0011606548796407878, "completions/clipped_ratio": 0.0, "completions/max_length": 527.0, "completions/max_terminated_length": 527.0, "completions/mean_length": 310.6600036621094, "completions/mean_terminated_length": 310.6600036621094, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 0.14149793088436127, "epoch": 0.23369565217391305, "frac_reward_zero_std": 0.0, "grad_norm": 0.01278267614543438, "learning_rate": 5e-05, "loss": 0.01356387510895729, "num_tokens": 5385701.0, "reward": 0.8483105301856995, "reward_std": 0.046941135078668594, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.15168945491313934, "rewards/length_penalty/std": 0.04694113880395889, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9945849776268005, "sampling/importance_sampling_ratio/min": 0.2063879370689392, "sampling/sampling_logp_difference/max": 1.5779976844787598, "sampling/sampling_logp_difference/mean": 0.02506762556731701, "step": 86, "step_time": 6.345105629879981 }, { "clip_ratio/high_max": 0.000682604851317592, "clip_ratio/high_mean": 0.000682604851317592, "clip_ratio/low_mean": 9.332875779364259e-05, "clip_ratio/low_min": 9.332875779364259e-05, "clip_ratio/region_mean": 0.0007759336091112346, "completions/clipped_ratio": 0.0, "completions/max_length": 1127.0, "completions/max_terminated_length": 1127.0, "completions/mean_length": 498.41998291015625, "completions/mean_terminated_length": 498.41998291015625, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 0.14586662650108337, "epoch": 0.23641304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.0155003247782588, "learning_rate": 5e-05, "loss": 0.029746606945991516, "num_tokens": 5412832.0, "reward": 0.3566308617591858, "reward_std": 0.560200035572052, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.24336914718151093, "rewards/length_penalty/std": 0.12490728497505188, "sampling/importance_sampling_ratio/max": 2.270101308822632, "sampling/importance_sampling_ratio/mean": 0.9942162036895752, "sampling/importance_sampling_ratio/min": 0.2935054302215576, "sampling/sampling_logp_difference/max": 1.2258591651916504, "sampling/sampling_logp_difference/mean": 0.01999354176223278, "step": 87, "step_time": 13.222105751745403 }, { "clip_ratio/high_max": 0.0005343808559700846, "clip_ratio/high_mean": 0.0005343808559700846, "clip_ratio/low_mean": 0.00016299273702315986, "clip_ratio/low_min": 0.00016299273702315986, "clip_ratio/region_mean": 0.0006973735755309463, "completions/clipped_ratio": 0.0, "completions/max_length": 679.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 382.0799865722656, "completions/mean_terminated_length": 382.0799865722656, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 0.13788711428642272, "epoch": 0.2391304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.016201354563236237, "learning_rate": 5e-05, "loss": 0.008741471916437149, "num_tokens": 5434826.0, "reward": 0.7734375, "reward_std": 0.2171802967786789, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.18656249344348907, "rewards/length_penalty/std": 0.06984082609415054, "sampling/importance_sampling_ratio/max": 2.7883358001708984, "sampling/importance_sampling_ratio/mean": 0.9944828152656555, "sampling/importance_sampling_ratio/min": 0.1857496201992035, "sampling/sampling_logp_difference/max": 1.6833555698394775, "sampling/sampling_logp_difference/mean": 0.021422382444143295, "step": 88, "step_time": 8.28874902217649 }, { "clip_ratio/high_max": 0.0007955357184982858, "clip_ratio/high_mean": 0.0007955357184982858, "clip_ratio/low_mean": 2.621919265948236e-05, "clip_ratio/low_min": 2.621919265948236e-05, "clip_ratio/region_mean": 0.0008217549111577682, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1909.0, "completions/mean_length": 618.5, "completions/mean_terminated_length": 589.3265380859375, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.17175524830818176, "epoch": 0.2418478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.013653785921633244, "learning_rate": 5e-05, "loss": 0.020935919135808945, "num_tokens": 5469211.0, "reward": 0.21799804270267487, "reward_std": 0.6661584377288818, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.302001953125, "rewards/length_penalty/std": 0.2297671139240265, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930328130722046, "sampling/importance_sampling_ratio/min": 0.303615540266037, "sampling/sampling_logp_difference/max": 1.191992998123169, "sampling/sampling_logp_difference/mean": 0.020726434886455536, "step": 89, "step_time": 22.4951800100971 }, { "clip_ratio/high_max": 0.0007658500340767205, "clip_ratio/high_mean": 0.0007658500340767205, "clip_ratio/low_mean": 2.3438416246790438e-05, "clip_ratio/low_min": 2.3438416246790438e-05, "clip_ratio/region_mean": 0.0007892884546890855, "completions/clipped_ratio": 0.0, "completions/max_length": 1998.0, "completions/max_terminated_length": 1998.0, "completions/mean_length": 659.739990234375, "completions/mean_terminated_length": 659.739990234375, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.14435225278139113, "epoch": 0.24456521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.019286051392555237, "learning_rate": 5e-05, "loss": 0.03661634027957916, "num_tokens": 5505228.0, "reward": 0.6178613305091858, "reward_std": 0.36470702290534973, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979514837265, "rewards/length_penalty/mean": -0.3221386671066284, "rewards/length_penalty/std": 0.22656019032001495, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9943234920501709, "sampling/importance_sampling_ratio/min": 0.3591086268424988, "sampling/sampling_logp_difference/max": 1.1295256614685059, "sampling/sampling_logp_difference/mean": 0.019089285284280777, "step": 90, "step_time": 21.950008247047663 }, { "clip_ratio/high_max": 0.0009108542930334806, "clip_ratio/high_mean": 0.0009108542930334806, "clip_ratio/low_mean": 0.00010948879062198103, "clip_ratio/low_min": 0.00010948879062198103, "clip_ratio/region_mean": 0.0010203430894762278, "completions/clipped_ratio": 0.0, "completions/max_length": 1604.0, "completions/max_terminated_length": 1604.0, "completions/mean_length": 517.0800170898438, "completions/mean_terminated_length": 517.0800170898438, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 0.17385907024145125, "epoch": 0.24728260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.017237825319170952, "learning_rate": 5e-05, "loss": 0.04393135383725166, "num_tokens": 5536252.0, "reward": 0.6875194907188416, "reward_std": 0.365161269903183, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979365825653, "rewards/length_penalty/mean": -0.25248047709465027, "rewards/length_penalty/std": 0.1945510357618332, "sampling/importance_sampling_ratio/max": 2.810533046722412, "sampling/importance_sampling_ratio/mean": 0.9928355813026428, "sampling/importance_sampling_ratio/min": 0.2914602756500244, "sampling/sampling_logp_difference/max": 1.232851505279541, "sampling/sampling_logp_difference/mean": 0.022177454084157944, "step": 91, "step_time": 18.435933465138078 }, { "clip_ratio/high_max": 0.00046310991747304796, "clip_ratio/high_mean": 0.00046310991747304796, "clip_ratio/low_mean": 0.00012931099627166986, "clip_ratio/low_min": 0.00012931099627166986, "clip_ratio/region_mean": 0.000592420919565484, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 511.2200012207031, "completions/mean_terminated_length": 479.8571472167969, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.14240270107984543, "epoch": 0.25, "frac_reward_zero_std": 0.0, "grad_norm": 0.014772976748645306, "learning_rate": 5e-05, "loss": 0.042228199541568756, "num_tokens": 5564203.0, "reward": 0.5503808259963989, "reward_std": 0.42025887966156006, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.24961914122104645, "rewards/length_penalty/std": 0.2091631144285202, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9942843317985535, "sampling/importance_sampling_ratio/min": 0.18323083221912384, "sampling/sampling_logp_difference/max": 1.6970086097717285, "sampling/sampling_logp_difference/mean": 0.021515555679798126, "step": 92, "step_time": 22.50430703512393 }, { "clip_ratio/high_max": 0.0007045186866889708, "clip_ratio/high_mean": 0.0007045186866889708, "clip_ratio/low_mean": 0.0002806035408866592, "clip_ratio/low_min": 0.0002806035408866592, "clip_ratio/region_mean": 0.0009851222421275452, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 1638.0, "completions/mean_length": 669.0999755859375, "completions/mean_terminated_length": 581.0850830078125, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "entropy": 0.20341256856918336, "epoch": 0.25271739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.020352164283394814, "learning_rate": 5e-05, "loss": 0.10169492661952972, "num_tokens": 5603278.0, "reward": 0.4732910096645355, "reward_std": 0.5773070454597473, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.32670897245407104, "rewards/length_penalty/std": 0.2420545071363449, "sampling/importance_sampling_ratio/max": 2.4024431705474854, "sampling/importance_sampling_ratio/mean": 0.9923534393310547, "sampling/importance_sampling_ratio/min": 0.2638809382915497, "sampling/sampling_logp_difference/max": 1.3322572708129883, "sampling/sampling_logp_difference/mean": 0.021952515468001366, "step": 93, "step_time": 23.539885837351903 }, { "clip_ratio/high_max": 0.0011616522213444113, "clip_ratio/high_mean": 0.0011616522213444113, "clip_ratio/low_mean": 0.0002181586460210383, "clip_ratio/low_min": 0.0002181586460210383, "clip_ratio/region_mean": 0.0013798108557239175, "completions/clipped_ratio": 0.0, "completions/max_length": 1176.0, "completions/max_terminated_length": 1176.0, "completions/mean_length": 450.239990234375, "completions/mean_terminated_length": 450.239990234375, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.19916799068450927, "epoch": 0.2554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.01750258356332779, "learning_rate": 5e-05, "loss": 0.0182659812271595, "num_tokens": 5631820.0, "reward": 0.30015623569488525, "reward_std": 0.461910218000412, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.21984374523162842, "rewards/length_penalty/std": 0.10487209260463715, "sampling/importance_sampling_ratio/max": 2.1481571197509766, "sampling/importance_sampling_ratio/mean": 0.991895854473114, "sampling/importance_sampling_ratio/min": 0.24272750318050385, "sampling/sampling_logp_difference/max": 1.415815830230713, "sampling/sampling_logp_difference/mean": 0.025588279590010643, "step": 94, "step_time": 13.953644986730069 }, { "clip_ratio/high_max": 0.0011509397649206222, "clip_ratio/high_mean": 0.0011509397649206222, "clip_ratio/low_mean": 0.00019863411434926093, "clip_ratio/low_min": 0.00019863411434926093, "clip_ratio/region_mean": 0.0013495738967321813, "completions/clipped_ratio": 0.0, "completions/max_length": 459.0, "completions/max_terminated_length": 459.0, "completions/mean_length": 290.0799865722656, "completions/mean_terminated_length": 290.0799865722656, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 0.1961733102798462, "epoch": 0.25815217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.01650562509894371, "learning_rate": 5e-05, "loss": 0.009883588179945946, "num_tokens": 5648694.0, "reward": 0.8383593559265137, "reward_std": 0.14430737495422363, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.14164061844348907, "rewards/length_penalty/std": 0.03877919539809227, "sampling/importance_sampling_ratio/max": 2.824411630630493, "sampling/importance_sampling_ratio/mean": 0.9931517839431763, "sampling/importance_sampling_ratio/min": 0.35756808519363403, "sampling/sampling_logp_difference/max": 1.0383000373840332, "sampling/sampling_logp_difference/mean": 0.030635839328169823, "step": 95, "step_time": 5.755662291543558 }, { "clip_ratio/high_max": 0.0006952405674383044, "clip_ratio/high_mean": 0.0006952405674383044, "clip_ratio/low_mean": 0.0003287168277893215, "clip_ratio/low_min": 0.0003287168277893215, "clip_ratio/region_mean": 0.0010239573777653277, "completions/clipped_ratio": 0.0, "completions/max_length": 684.0, "completions/max_terminated_length": 684.0, "completions/mean_length": 312.79998779296875, "completions/mean_terminated_length": 312.79998779296875, "completions/min_length": 101.0, "completions/min_terminated_length": 101.0, "entropy": 0.18830790221691132, "epoch": 0.2608695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.0161149799823761, "learning_rate": 5e-05, "loss": 0.014134292490780354, "num_tokens": 5667864.0, "reward": 0.5272656083106995, "reward_std": 0.5117128491401672, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.15273436903953552, "rewards/length_penalty/std": 0.0754011869430542, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9926605820655823, "sampling/importance_sampling_ratio/min": 0.2316138744354248, "sampling/sampling_logp_difference/max": 1.4626836776733398, "sampling/sampling_logp_difference/mean": 0.0293266698718071, "step": 96, "step_time": 8.08534470712766 }, { "clip_ratio/high_max": 0.0009365640406031161, "clip_ratio/high_mean": 0.0009365640406031161, "clip_ratio/low_mean": 7.815553108230233e-05, "clip_ratio/low_min": 7.815553108230233e-05, "clip_ratio/region_mean": 0.0010147195833269506, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 302.9599914550781, "completions/mean_terminated_length": 302.9599914550781, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 0.19039133489131926, "epoch": 0.26358695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.01323253195732832, "learning_rate": 5e-05, "loss": 0.007963886484503746, "num_tokens": 5685942.0, "reward": 0.8320702910423279, "reward_std": 0.14255400002002716, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.14792968332767487, "rewards/length_penalty/std": 0.053403496742248535, "sampling/importance_sampling_ratio/max": 2.9804489612579346, "sampling/importance_sampling_ratio/mean": 0.9915622472763062, "sampling/importance_sampling_ratio/min": 0.23956701159477234, "sampling/sampling_logp_difference/max": 1.428922176361084, "sampling/sampling_logp_difference/mean": 0.029504768550395966, "step": 97, "step_time": 7.233770550927147 }, { "clip_ratio/high_max": 0.0015867566457018257, "clip_ratio/high_mean": 0.0015867566457018257, "clip_ratio/low_mean": 0.00011946987069677562, "clip_ratio/low_min": 0.00011946987069677562, "clip_ratio/region_mean": 0.0017062265193089842, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 1352.0, "completions/mean_length": 349.7599792480469, "completions/mean_terminated_length": 241.3616943359375, "completions/min_length": 74.0, "completions/min_terminated_length": 74.0, "entropy": 0.22566365003585814, "epoch": 0.266304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.015148256905376911, "learning_rate": 5e-05, "loss": 0.0725148394703865, "num_tokens": 5705570.0, "reward": 0.2292187511920929, "reward_std": 0.6269381046295166, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.17078125476837158, "rewards/length_penalty/std": 0.24175436794757843, "sampling/importance_sampling_ratio/max": 2.5255091190338135, "sampling/importance_sampling_ratio/mean": 0.991366982460022, "sampling/importance_sampling_ratio/min": 0.09896191209554672, "sampling/sampling_logp_difference/max": 2.3130202293395996, "sampling/sampling_logp_difference/mean": 0.026597710326313972, "step": 98, "step_time": 21.33166946657002 }, { "clip_ratio/high_max": 0.0010735183954238892, "clip_ratio/high_mean": 0.0010735183954238892, "clip_ratio/low_mean": 0.00030840777326375247, "clip_ratio/low_min": 0.00030840777326375247, "clip_ratio/region_mean": 0.0013819261686876417, "completions/clipped_ratio": 0.0, "completions/max_length": 310.0, "completions/max_terminated_length": 310.0, "completions/mean_length": 185.86000061035156, "completions/mean_terminated_length": 185.86000061035156, "completions/min_length": 92.0, "completions/min_terminated_length": 92.0, "entropy": 0.20395126342773437, "epoch": 0.26902173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.012664223089814186, "learning_rate": 5e-05, "loss": 0.007070655468851328, "num_tokens": 5717773.0, "reward": 0.6292480230331421, "reward_std": 0.46103474497795105, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.09075195342302322, "rewards/length_penalty/std": 0.024525616317987442, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9933871626853943, "sampling/importance_sampling_ratio/min": 0.20495867729187012, "sampling/sampling_logp_difference/max": 1.584946870803833, "sampling/sampling_logp_difference/mean": 0.033781420439481735, "step": 99, "step_time": 4.117180567001924 }, { "clip_ratio/high_max": 0.0005707273143343627, "clip_ratio/high_mean": 0.0005707273143343627, "clip_ratio/low_mean": 9.760858956724406e-05, "clip_ratio/low_min": 9.760858956724406e-05, "clip_ratio/region_mean": 0.0006683359039016068, "completions/clipped_ratio": 0.0, "completions/max_length": 346.0, "completions/max_terminated_length": 346.0, "completions/mean_length": 187.25999450683594, "completions/mean_terminated_length": 187.25999450683594, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "entropy": 0.17016780972480774, "epoch": 0.2717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.012072809040546417, "learning_rate": 5e-05, "loss": 0.008675863966345787, "num_tokens": 5729366.0, "reward": 0.88856440782547, "reward_std": 0.14345325529575348, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.09143554419279099, "rewards/length_penalty/std": 0.03134666755795479, "sampling/importance_sampling_ratio/max": 2.592597246170044, "sampling/importance_sampling_ratio/mean": 0.991979718208313, "sampling/importance_sampling_ratio/min": 0.28697964549064636, "sampling/sampling_logp_difference/max": 1.2483439445495605, "sampling/sampling_logp_difference/mean": 0.03182890638709068, "step": 100, "step_time": 4.314533639000729 }, { "clip_ratio/high_max": 0.0012083934037946165, "clip_ratio/high_mean": 0.0012083934037946165, "clip_ratio/low_mean": 0.00026164305745624006, "clip_ratio/low_min": 0.00026164305745624006, "clip_ratio/region_mean": 0.0014700364205054938, "completions/clipped_ratio": 0.0, "completions/max_length": 452.0, "completions/max_terminated_length": 452.0, "completions/mean_length": 204.6599884033203, "completions/mean_terminated_length": 204.6599884033203, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "entropy": 0.19577673971652984, "epoch": 0.27445652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.013354828581213951, "learning_rate": 5e-05, "loss": 0.012918182648718357, "num_tokens": 5742489.0, "reward": 0.8400683403015137, "reward_std": 0.24915407598018646, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979365825653, "rewards/length_penalty/mean": -0.09993164241313934, "rewards/length_penalty/std": 0.04284634813666344, "sampling/importance_sampling_ratio/max": 2.8144924640655518, "sampling/importance_sampling_ratio/mean": 0.9925035834312439, "sampling/importance_sampling_ratio/min": 0.19658847153186798, "sampling/sampling_logp_difference/max": 1.6266427040100098, "sampling/sampling_logp_difference/mean": 0.03311571851372719, "step": 101, "step_time": 5.483416471164674 }, { "clip_ratio/high_max": 0.0009411082835868001, "clip_ratio/high_mean": 0.0009411082835868001, "clip_ratio/low_mean": 0.00022402363829314708, "clip_ratio/low_min": 0.00022402363829314708, "clip_ratio/region_mean": 0.0011651319218799472, "completions/clipped_ratio": 0.0, "completions/max_length": 796.0, "completions/max_terminated_length": 796.0, "completions/mean_length": 234.13999938964844, "completions/mean_terminated_length": 234.13999938964844, "completions/min_length": 50.0, "completions/min_terminated_length": 50.0, "entropy": 0.2318212479352951, "epoch": 0.27717391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.014598333276808262, "learning_rate": 5e-05, "loss": -0.0033794669434428215, "num_tokens": 5757126.0, "reward": 0.6456738114356995, "reward_std": 0.48349666595458984, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.11432617157697678, "rewards/length_penalty/std": 0.08932963013648987, "sampling/importance_sampling_ratio/max": 2.6214189529418945, "sampling/importance_sampling_ratio/mean": 0.989772379398346, "sampling/importance_sampling_ratio/min": 0.1754714399576187, "sampling/sampling_logp_difference/max": 1.740278959274292, "sampling/sampling_logp_difference/mean": 0.034885525703430176, "step": 102, "step_time": 8.834860250819474 }, { "clip_ratio/high_max": 0.0014411504263989628, "clip_ratio/high_mean": 0.0014411504263989628, "clip_ratio/low_mean": 9.528346708975732e-05, "clip_ratio/low_min": 9.528346708975732e-05, "clip_ratio/region_mean": 0.0015364338527433575, "completions/clipped_ratio": 0.0, "completions/max_length": 348.0, "completions/max_terminated_length": 348.0, "completions/mean_length": 174.37998962402344, "completions/mean_terminated_length": 174.37998962402344, "completions/min_length": 71.0, "completions/min_terminated_length": 71.0, "entropy": 0.2715637177228928, "epoch": 0.2798913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.017685387283563614, "learning_rate": 5e-05, "loss": 0.01238732598721981, "num_tokens": 5770085.0, "reward": 0.7148535251617432, "reward_std": 0.427943617105484, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.08514648675918579, "rewards/length_penalty/std": 0.03423784300684929, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9892432689666748, "sampling/importance_sampling_ratio/min": 0.2573729157447815, "sampling/sampling_logp_difference/max": 1.357229232788086, "sampling/sampling_logp_difference/mean": 0.038839783519506454, "step": 103, "step_time": 4.841475100722164 }, { "clip_ratio/high_max": 0.0018297017086297274, "clip_ratio/high_mean": 0.0018297017086297274, "clip_ratio/low_mean": 0.00037032614927738904, "clip_ratio/low_min": 0.00037032614927738904, "clip_ratio/region_mean": 0.002200027834624052, "completions/clipped_ratio": 0.0, "completions/max_length": 930.0, "completions/max_terminated_length": 930.0, "completions/mean_length": 327.6000061035156, "completions/mean_terminated_length": 327.6000061035156, "completions/min_length": 79.0, "completions/min_terminated_length": 79.0, "entropy": 0.29253464937210083, "epoch": 0.2826086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.017163075506687164, "learning_rate": 5e-05, "loss": 0.02614499069750309, "num_tokens": 5789975.0, "reward": 0.2000390589237213, "reward_std": 0.5057170987129211, "rewards/correctness/mean": 0.36000001430511475, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.15996094048023224, "rewards/length_penalty/std": 0.09843949228525162, "sampling/importance_sampling_ratio/max": 2.5560290813446045, "sampling/importance_sampling_ratio/mean": 0.9881027936935425, "sampling/importance_sampling_ratio/min": 0.21785800158977509, "sampling/sampling_logp_difference/max": 1.5239118337631226, "sampling/sampling_logp_difference/mean": 0.03395746275782585, "step": 104, "step_time": 10.365797318983823 }, { "clip_ratio/high_max": 0.0016505021136254071, "clip_ratio/high_mean": 0.0016505021136254071, "clip_ratio/low_mean": 0.0004149336367845535, "clip_ratio/low_min": 0.0004149336367845535, "clip_ratio/region_mean": 0.0020654357504099607, "completions/clipped_ratio": 0.0, "completions/max_length": 525.0, "completions/max_terminated_length": 525.0, "completions/mean_length": 185.8199920654297, "completions/mean_terminated_length": 185.8199920654297, "completions/min_length": 76.0, "completions/min_terminated_length": 76.0, "entropy": 0.23983452320098878, "epoch": 0.28532608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.012423444539308548, "learning_rate": 5e-05, "loss": 0.002859638072550297, "num_tokens": 5801626.0, "reward": 0.38926756381988525, "reward_std": 0.5165557265281677, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.09073242545127869, "rewards/length_penalty/std": 0.04708758741617203, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9901142716407776, "sampling/importance_sampling_ratio/min": 0.17719721794128418, "sampling/sampling_logp_difference/max": 1.7304918766021729, "sampling/sampling_logp_difference/mean": 0.03442460298538208, "step": 105, "step_time": 6.492382062831894 }, { "clip_ratio/high_max": 0.0005780933774076402, "clip_ratio/high_mean": 0.0005780933774076402, "clip_ratio/low_mean": 0.0005185882211662829, "clip_ratio/low_min": 0.0005185882211662829, "clip_ratio/region_mean": 0.001096681598573923, "completions/clipped_ratio": 0.0, "completions/max_length": 565.0, "completions/max_terminated_length": 565.0, "completions/mean_length": 158.51998901367188, "completions/mean_terminated_length": 158.51998901367188, "completions/min_length": 64.0, "completions/min_terminated_length": 64.0, "entropy": 0.22457527220249177, "epoch": 0.28804347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.014758552424609661, "learning_rate": 5e-05, "loss": -0.0042253821156919, "num_tokens": 5812032.0, "reward": 0.7225976586341858, "reward_std": 0.43400877714157104, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.07740234583616257, "rewards/length_penalty/std": 0.054773226380348206, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9911705255508423, "sampling/importance_sampling_ratio/min": 0.29393672943115234, "sampling/sampling_logp_difference/max": 1.2798686027526855, "sampling/sampling_logp_difference/mean": 0.03353006765246391, "step": 106, "step_time": 6.204408699879423 }, { "clip_ratio/high_max": 0.0013835940160788595, "clip_ratio/high_mean": 0.0013835940160788595, "clip_ratio/low_mean": 0.0005407711723819375, "clip_ratio/low_min": 0.0005407711723819375, "clip_ratio/region_mean": 0.0019243651768192649, "completions/clipped_ratio": 0.0, "completions/max_length": 1896.0, "completions/max_terminated_length": 1896.0, "completions/mean_length": 377.41998291015625, "completions/mean_terminated_length": 377.41998291015625, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "entropy": 0.3654423654079437, "epoch": 0.2907608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.016043387353420258, "learning_rate": 5e-05, "loss": 0.030646812170743942, "num_tokens": 5834943.0, "reward": 0.33571287989616394, "reward_std": 0.6409838795661926, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.18428711593151093, "rewards/length_penalty/std": 0.21686908602714539, "sampling/importance_sampling_ratio/max": 2.1414754390716553, "sampling/importance_sampling_ratio/mean": 0.9864636063575745, "sampling/importance_sampling_ratio/min": 0.16718970239162445, "sampling/sampling_logp_difference/max": 1.7886261940002441, "sampling/sampling_logp_difference/mean": 0.03430148586630821, "step": 107, "step_time": 20.311360628111288 }, { "clip_ratio/high_max": 0.0022310700500383974, "clip_ratio/high_mean": 0.0022310700500383974, "clip_ratio/low_mean": 0.0008199853007681667, "clip_ratio/low_min": 0.0008199853007681667, "clip_ratio/region_mean": 0.003051055292598903, "completions/clipped_ratio": 0.0, "completions/max_length": 315.0, "completions/max_terminated_length": 315.0, "completions/mean_length": 151.67999267578125, "completions/mean_terminated_length": 151.67999267578125, "completions/min_length": 60.0, "completions/min_terminated_length": 60.0, "entropy": 0.2812129735946655, "epoch": 0.29347826086956524, "frac_reward_zero_std": 0.0, "grad_norm": 0.01154936384409666, "learning_rate": 5e-05, "loss": 0.00485996576026082, "num_tokens": 5845577.0, "reward": 0.4059374928474426, "reward_std": 0.5127573013305664, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.07406249642372131, "rewards/length_penalty/std": 0.028034603223204613, "sampling/importance_sampling_ratio/max": 2.603525161743164, "sampling/importance_sampling_ratio/mean": 0.9909453988075256, "sampling/importance_sampling_ratio/min": 0.1587897688150406, "sampling/sampling_logp_difference/max": 1.8401741981506348, "sampling/sampling_logp_difference/mean": 0.04085557535290718, "step": 108, "step_time": 4.2213947479613125 }, { "clip_ratio/high_max": 0.001480993302538991, "clip_ratio/high_mean": 0.001480993302538991, "clip_ratio/low_mean": 0.00021252454607747495, "clip_ratio/low_min": 0.00021252454607747495, "clip_ratio/region_mean": 0.0016935178078711032, "completions/clipped_ratio": 0.0, "completions/max_length": 1066.0, "completions/max_terminated_length": 1066.0, "completions/mean_length": 164.4199981689453, "completions/mean_terminated_length": 164.4199981689453, "completions/min_length": 67.0, "completions/min_terminated_length": 67.0, "entropy": 0.31263987720012665, "epoch": 0.296195652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.012847491540014744, "learning_rate": 5e-05, "loss": 0.010743032209575176, "num_tokens": 5856138.0, "reward": 0.5997167825698853, "reward_std": 0.520103394985199, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.08028320223093033, "rewards/length_penalty/std": 0.0816592127084732, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9857426881790161, "sampling/importance_sampling_ratio/min": 0.2710735499858856, "sampling/sampling_logp_difference/max": 1.3053652048110962, "sampling/sampling_logp_difference/mean": 0.04090895876288414, "step": 109, "step_time": 10.641943448223174 }, { "clip_ratio/high_max": 0.0023372828494757415, "clip_ratio/high_mean": 0.0023372828494757415, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0023372828494757415, "completions/clipped_ratio": 0.0, "completions/max_length": 224.0, "completions/max_terminated_length": 224.0, "completions/mean_length": 121.57999420166016, "completions/mean_terminated_length": 121.57999420166016, "completions/min_length": 42.0, "completions/min_terminated_length": 42.0, "entropy": 0.25978425741195676, "epoch": 0.29891304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.012036431580781937, "learning_rate": 5e-05, "loss": 0.0021082288585603237, "num_tokens": 5864797.0, "reward": 0.8406347632408142, "reward_std": 0.3036022186279297, "rewards/correctness/mean": 0.8999999761581421, "rewards/correctness/std": 0.30304577946662903, "rewards/length_penalty/mean": -0.05936523526906967, "rewards/length_penalty/std": 0.025582697242498398, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9901321530342102, "sampling/importance_sampling_ratio/min": 0.21720051765441895, "sampling/sampling_logp_difference/max": 1.5269343852996826, "sampling/sampling_logp_difference/mean": 0.04141371324658394, "step": 110, "step_time": 3.1361711190547794 }, { "clip_ratio/high_max": 0.0003656261716969311, "clip_ratio/high_mean": 0.0003656261716969311, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0003656261716969311, "completions/clipped_ratio": 0.0, "completions/max_length": 285.0, "completions/max_terminated_length": 285.0, "completions/mean_length": 98.81999969482422, "completions/mean_terminated_length": 98.81999969482422, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.19361501634120942, "epoch": 0.3016304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.012895033694803715, "learning_rate": 5e-05, "loss": 0.00398594792932272, "num_tokens": 5872398.0, "reward": 0.6717480421066284, "reward_std": 0.45583122968673706, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.04825195297598839, "rewards/length_penalty/std": 0.02495020627975464, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.992006242275238, "sampling/importance_sampling_ratio/min": 0.15148277580738068, "sampling/sampling_logp_difference/max": 1.8872833251953125, "sampling/sampling_logp_difference/mean": 0.03226413577795029, "step": 111, "step_time": 3.619885540334508 }, { "clip_ratio/high_max": 0.0004390946705825627, "clip_ratio/high_mean": 0.0004390946705825627, "clip_ratio/low_mean": 0.00033824756974354386, "clip_ratio/low_min": 0.00033824756974354386, "clip_ratio/region_mean": 0.0007773422403261065, "completions/clipped_ratio": 0.0, "completions/max_length": 578.0, "completions/max_terminated_length": 578.0, "completions/mean_length": 136.27999877929688, "completions/mean_terminated_length": 136.27999877929688, "completions/min_length": 50.0, "completions/min_terminated_length": 50.0, "entropy": 0.32020220458507537, "epoch": 0.30434782608695654, "frac_reward_zero_std": 0.0, "grad_norm": 0.012484963983297348, "learning_rate": 5e-05, "loss": 0.009005909785628319, "num_tokens": 5881512.0, "reward": 0.31345701217651367, "reward_std": 0.5048692226409912, "rewards/correctness/mean": 0.3799999952316284, "rewards/correctness/std": 0.4903143346309662, "rewards/length_penalty/mean": -0.06654296815395355, "rewards/length_penalty/std": 0.05672767013311386, "sampling/importance_sampling_ratio/max": 2.9808573722839355, "sampling/importance_sampling_ratio/mean": 0.9861035346984863, "sampling/importance_sampling_ratio/min": 0.22736811637878418, "sampling/sampling_logp_difference/max": 1.481184959411621, "sampling/sampling_logp_difference/mean": 0.037614136934280396, "step": 112, "step_time": 6.328286922769621 }, { "clip_ratio/high_max": 0.0008063952205702663, "clip_ratio/high_mean": 0.0008063952205702663, "clip_ratio/low_mean": 0.0002684563864022493, "clip_ratio/low_min": 0.0002684563864022493, "clip_ratio/region_mean": 0.0010748516069725156, "completions/clipped_ratio": 0.0, "completions/max_length": 159.0, "completions/max_terminated_length": 159.0, "completions/mean_length": 76.43999481201172, "completions/mean_terminated_length": 76.43999481201172, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 0.2393881142139435, "epoch": 0.3070652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.009909100830554962, "learning_rate": 5e-05, "loss": 0.0011619285214692354, "num_tokens": 5888644.0, "reward": 0.6426757574081421, "reward_std": 0.47860875725746155, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.037324219942092896, "rewards/length_penalty/std": 0.013797693885862827, "sampling/importance_sampling_ratio/max": 2.7209415435791016, "sampling/importance_sampling_ratio/mean": 0.9932952523231506, "sampling/importance_sampling_ratio/min": 0.3156265318393707, "sampling/sampling_logp_difference/max": 1.1531956195831299, "sampling/sampling_logp_difference/mean": 0.03954983130097389, "step": 113, "step_time": 2.577970744809136 }, { "clip_ratio/high_max": 0.0017212016391567886, "clip_ratio/high_mean": 0.0017212016391567886, "clip_ratio/low_mean": 0.00017621145816519855, "clip_ratio/low_min": 0.00017621145816519855, "clip_ratio/region_mean": 0.0018974130623973906, "completions/clipped_ratio": 0.0, "completions/max_length": 359.0, "completions/max_terminated_length": 359.0, "completions/mean_length": 99.95999908447266, "completions/mean_terminated_length": 99.95999908447266, "completions/min_length": 46.0, "completions/min_terminated_length": 46.0, "entropy": 0.27610541582107545, "epoch": 0.30978260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.014195474795997143, "learning_rate": 5e-05, "loss": 0.0016526561230421066, "num_tokens": 5896182.0, "reward": 0.6311913728713989, "reward_std": 0.48117876052856445, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.048808593302965164, "rewards/length_penalty/std": 0.027750348672270775, "sampling/importance_sampling_ratio/max": 1.9509680271148682, "sampling/importance_sampling_ratio/mean": 0.9879252314567566, "sampling/importance_sampling_ratio/min": 0.34516289830207825, "sampling/sampling_logp_difference/max": 1.0637388229370117, "sampling/sampling_logp_difference/mean": 0.036363665014505386, "step": 114, "step_time": 4.703229781938717 }, { "clip_ratio/high_max": 0.002066467539407313, "clip_ratio/high_mean": 0.002066467539407313, "clip_ratio/low_mean": 0.00025575447361916306, "clip_ratio/low_min": 0.00025575447361916306, "clip_ratio/region_mean": 0.002322222013026476, "completions/clipped_ratio": 0.0, "completions/max_length": 176.0, "completions/max_terminated_length": 176.0, "completions/mean_length": 80.36000061035156, "completions/mean_terminated_length": 80.36000061035156, "completions/min_length": 37.0, "completions/min_terminated_length": 37.0, "entropy": 0.22362540960311889, "epoch": 0.3125, "frac_reward_zero_std": 0.0, "grad_norm": 0.01079811342060566, "learning_rate": 5e-05, "loss": -0.002070262096822262, "num_tokens": 5902430.0, "reward": 0.580761730670929, "reward_std": 0.49483081698417664, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.039238281548023224, "rewards/length_penalty/std": 0.014868262223899364, "sampling/importance_sampling_ratio/max": 2.2997214794158936, "sampling/importance_sampling_ratio/mean": 0.9913575053215027, "sampling/importance_sampling_ratio/min": 0.2111196517944336, "sampling/sampling_logp_difference/max": 1.5553302764892578, "sampling/sampling_logp_difference/mean": 0.034344468265771866, "step": 115, "step_time": 2.6515860850922763 }, { "clip_ratio/high_max": 0.0019970620749518277, "clip_ratio/high_mean": 0.0019970620749518277, "clip_ratio/low_mean": 0.0002793296007439494, "clip_ratio/low_min": 0.0002793296007439494, "clip_ratio/region_mean": 0.00227639174554497, "completions/clipped_ratio": 0.0, "completions/max_length": 180.0, "completions/max_terminated_length": 180.0, "completions/mean_length": 78.41999816894531, "completions/mean_terminated_length": 78.41999816894531, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.24196649491786956, "epoch": 0.31521739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.013271356001496315, "learning_rate": 5e-05, "loss": 0.0022566956467926502, "num_tokens": 5909241.0, "reward": 0.8017089366912842, "reward_std": 0.37205037474632263, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.03829101473093033, "rewards/length_penalty/std": 0.015793830156326294, "sampling/importance_sampling_ratio/max": 2.9569191932678223, "sampling/importance_sampling_ratio/mean": 0.9928051829338074, "sampling/importance_sampling_ratio/min": 0.2952917516231537, "sampling/sampling_logp_difference/max": 1.2197914123535156, "sampling/sampling_logp_difference/mean": 0.039991345256567, "step": 116, "step_time": 2.655353401787579 }, { "clip_ratio/high_max": 0.0033249396830797194, "clip_ratio/high_mean": 0.0033249396830797194, "clip_ratio/low_mean": 0.0013393433764576912, "clip_ratio/low_min": 0.0013393433764576912, "clip_ratio/region_mean": 0.0046642830595374106, "completions/clipped_ratio": 0.0, "completions/max_length": 67.0, "completions/max_terminated_length": 67.0, "completions/mean_length": 43.599998474121094, "completions/mean_terminated_length": 43.599998474121094, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "entropy": 0.2081553041934967, "epoch": 0.3179347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.008443817496299744, "learning_rate": 5e-05, "loss": 0.0012317131040617824, "num_tokens": 5913601.0, "reward": 0.7587109208106995, "reward_std": 0.4203566610813141, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.02128906175494194, "rewards/length_penalty/std": 0.005932000931352377, "sampling/importance_sampling_ratio/max": 1.8893288373947144, "sampling/importance_sampling_ratio/mean": 0.9916374087333679, "sampling/importance_sampling_ratio/min": 0.27941209077835083, "sampling/sampling_logp_difference/max": 1.2750675678253174, "sampling/sampling_logp_difference/mean": 0.033489543944597244, "step": 117, "step_time": 1.8823761746753007 }, { "clip_ratio/high_max": 0.0005688481265679002, "clip_ratio/high_mean": 0.0005688481265679002, "clip_ratio/low_mean": 0.0012103871908038855, "clip_ratio/low_min": 0.0012103871908038855, "clip_ratio/region_mean": 0.0017792353173717856, "completions/clipped_ratio": 0.0, "completions/max_length": 142.0, "completions/max_terminated_length": 142.0, "completions/mean_length": 72.47999572753906, "completions/mean_terminated_length": 72.47999572753906, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 0.27200756371021273, "epoch": 0.32065217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.009959971532225609, "learning_rate": 5e-05, "loss": 0.001126274699345231, "num_tokens": 5920905.0, "reward": 0.6646093726158142, "reward_std": 0.4720146656036377, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.03539062663912773, "rewards/length_penalty/std": 0.015659291297197342, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9858464598655701, "sampling/importance_sampling_ratio/min": 0.17056792974472046, "sampling/sampling_logp_difference/max": 1.7686216831207275, "sampling/sampling_logp_difference/mean": 0.04525027051568031, "step": 118, "step_time": 2.567681823624298 }, { "clip_ratio/high_max": 0.0012298159534111619, "clip_ratio/high_mean": 0.0012298159534111619, "clip_ratio/low_mean": 0.0004323530476540327, "clip_ratio/low_min": 0.0004323530476540327, "clip_ratio/region_mean": 0.0016621690359897912, "completions/clipped_ratio": 0.0, "completions/max_length": 1335.0, "completions/max_terminated_length": 1335.0, "completions/mean_length": 139.77999877929688, "completions/mean_terminated_length": 139.77999877929688, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 0.3178446352481842, "epoch": 0.3233695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.012043810449540615, "learning_rate": 5e-05, "loss": 0.011804413050413132, "num_tokens": 5931404.0, "reward": 0.4517480432987213, "reward_std": 0.5475705862045288, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.06825195252895355, "rewards/length_penalty/std": 0.0961272343993187, "sampling/importance_sampling_ratio/max": 2.9376678466796875, "sampling/importance_sampling_ratio/mean": 0.9891913533210754, "sampling/importance_sampling_ratio/min": 0.1745537668466568, "sampling/sampling_logp_difference/max": 1.7455224990844727, "sampling/sampling_logp_difference/mean": 0.03469213470816612, "step": 119, "step_time": 13.277865092735738 }, { "clip_ratio/high_max": 0.0007105602882802487, "clip_ratio/high_mean": 0.0007105602882802487, "clip_ratio/low_mean": 0.0009997313376516105, "clip_ratio/low_min": 0.0009997313376516105, "clip_ratio/region_mean": 0.001710291625931859, "completions/clipped_ratio": 0.0, "completions/max_length": 113.0, "completions/max_terminated_length": 113.0, "completions/mean_length": 57.2599983215332, "completions/mean_terminated_length": 57.2599983215332, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.28369104862213135, "epoch": 0.32608695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.014030136168003082, "learning_rate": 5e-05, "loss": 0.0005326682003214955, "num_tokens": 5937117.0, "reward": 0.6920409798622131, "reward_std": 0.45576512813568115, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573422908783, "rewards/length_penalty/mean": -0.027958983555436134, "rewards/length_penalty/std": 0.011268665082752705, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9907137155532837, "sampling/importance_sampling_ratio/min": 0.3402984142303467, "sampling/sampling_logp_difference/max": 1.1216139793395996, "sampling/sampling_logp_difference/mean": 0.04103627800941467, "step": 120, "step_time": 2.1914469429757446 }, { "clip_ratio/high_max": 0.002339954860508442, "clip_ratio/high_mean": 0.002339954860508442, "clip_ratio/low_mean": 0.002092923759482801, "clip_ratio/low_min": 0.002092923759482801, "clip_ratio/region_mean": 0.004432878619991243, "completions/clipped_ratio": 0.0, "completions/max_length": 258.0, "completions/max_terminated_length": 258.0, "completions/mean_length": 77.43999481201172, "completions/mean_terminated_length": 77.43999481201172, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "entropy": 0.4044592082500458, "epoch": 0.328804347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.013229206204414368, "learning_rate": 5e-05, "loss": 0.007325820624828339, "num_tokens": 5944329.0, "reward": 0.5021874904632568, "reward_std": 0.5180118083953857, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.037812501192092896, "rewards/length_penalty/std": 0.029638368636369705, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9860542416572571, "sampling/importance_sampling_ratio/min": 0.10667832940816879, "sampling/sampling_logp_difference/max": 2.2379372119903564, "sampling/sampling_logp_difference/mean": 0.04855887591838837, "step": 121, "step_time": 3.5320951712783426 }, { "clip_ratio/high_max": 0.0020871169865131376, "clip_ratio/high_mean": 0.0020871169865131376, "clip_ratio/low_mean": 0.0006550624966621399, "clip_ratio/low_min": 0.0006550624966621399, "clip_ratio/region_mean": 0.0027421795064583422, "completions/clipped_ratio": 0.0, "completions/max_length": 342.0, "completions/max_terminated_length": 342.0, "completions/mean_length": 62.63999938964844, "completions/mean_terminated_length": 62.63999938964844, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 0.3629956305027008, "epoch": 0.33152173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.011344057507812977, "learning_rate": 5e-05, "loss": 0.003215065225958824, "num_tokens": 5950581.0, "reward": 0.4694140553474426, "reward_std": 0.5132778882980347, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.030585937201976776, "rewards/length_penalty/std": 0.023655664175748825, "sampling/importance_sampling_ratio/max": 2.0439770221710205, "sampling/importance_sampling_ratio/mean": 0.9834688901901245, "sampling/importance_sampling_ratio/min": 0.33990684151649475, "sampling/sampling_logp_difference/max": 1.0790836811065674, "sampling/sampling_logp_difference/mean": 0.043290577828884125, "step": 122, "step_time": 3.9696634637657553 }, { "clip_ratio/high_max": 0.0021428524632938206, "clip_ratio/high_mean": 0.0021428524632938206, "clip_ratio/low_mean": 0.0003522538812831044, "clip_ratio/low_min": 0.0003522538812831044, "clip_ratio/region_mean": 0.0024951063678599894, "completions/clipped_ratio": 0.0, "completions/max_length": 1630.0, "completions/max_terminated_length": 1630.0, "completions/mean_length": 105.41999816894531, "completions/mean_terminated_length": 105.41999816894531, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.25871089696884153, "epoch": 0.3342391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.019554821774363518, "learning_rate": 5e-05, "loss": 0.02193872630596161, "num_tokens": 5959092.0, "reward": 0.34852537512779236, "reward_std": 0.5240065455436707, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.05147460848093033, "rewards/length_penalty/std": 0.1117917150259018, "sampling/importance_sampling_ratio/max": 2.471583604812622, "sampling/importance_sampling_ratio/mean": 0.9906110167503357, "sampling/importance_sampling_ratio/min": 0.1870785504579544, "sampling/sampling_logp_difference/max": 1.6762266159057617, "sampling/sampling_logp_difference/mean": 0.029479123651981354, "step": 123, "step_time": 15.582083855988458 }, { "clip_ratio/high_max": 0.00041841003112494943, "clip_ratio/high_mean": 0.00041841003112494943, "clip_ratio/low_mean": 0.00041841003112494943, "clip_ratio/low_min": 0.00041841003112494943, "clip_ratio/region_mean": 0.0008368200622498989, "completions/clipped_ratio": 0.0, "completions/max_length": 103.0, "completions/max_terminated_length": 103.0, "completions/mean_length": 42.70000076293945, "completions/mean_terminated_length": 42.70000076293945, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.18871377408504486, "epoch": 0.33695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.012075711973011494, "learning_rate": 5e-05, "loss": 0.003170022275298834, "num_tokens": 5963477.0, "reward": 0.5391503572463989, "reward_std": 0.5063897967338562, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.02084960974752903, "rewards/length_penalty/std": 0.011277487501502037, "sampling/importance_sampling_ratio/max": 2.757899284362793, "sampling/importance_sampling_ratio/mean": 0.9949593544006348, "sampling/importance_sampling_ratio/min": 0.26883986592292786, "sampling/sampling_logp_difference/max": 1.3136394023895264, "sampling/sampling_logp_difference/mean": 0.035366520285606384, "step": 124, "step_time": 2.537050310987979 }, { "clip_ratio/high_max": 0.0037846322637051346, "clip_ratio/high_mean": 0.0037846322637051346, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037846322637051346, "completions/clipped_ratio": 0.0, "completions/max_length": 89.0, "completions/max_terminated_length": 89.0, "completions/mean_length": 43.02000045776367, "completions/mean_terminated_length": 43.02000045776367, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "entropy": 0.2748114734888077, "epoch": 0.33967391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.010747011750936508, "learning_rate": 5e-05, "loss": -1.1701034964062274e-05, "num_tokens": 5968108.0, "reward": 0.47899413108825684, "reward_std": 0.5066565275192261, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.021005859598517418, "rewards/length_penalty/std": 0.008610362187027931, "sampling/importance_sampling_ratio/max": 1.9138907194137573, "sampling/importance_sampling_ratio/mean": 0.990419864654541, "sampling/importance_sampling_ratio/min": 0.3262421786785126, "sampling/sampling_logp_difference/max": 1.1201152801513672, "sampling/sampling_logp_difference/mean": 0.04402812942862511, "step": 125, "step_time": 1.9398821538779885 }, { "clip_ratio/high_max": 0.0012025600532069801, "clip_ratio/high_mean": 0.0012025600532069801, "clip_ratio/low_mean": 0.0009474263759329915, "clip_ratio/low_min": 0.0009474263759329915, "clip_ratio/region_mean": 0.002149986429139972, "completions/clipped_ratio": 0.0, "completions/max_length": 247.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 61.03999710083008, "completions/mean_terminated_length": 61.03999710083008, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "entropy": 0.3201991945505142, "epoch": 0.3423913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.013153769075870514, "learning_rate": 5e-05, "loss": 0.008743176236748695, "num_tokens": 5976130.0, "reward": 0.47019529342651367, "reward_std": 0.5109017491340637, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.029804687947034836, "rewards/length_penalty/std": 0.024028556421399117, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9863394498825073, "sampling/importance_sampling_ratio/min": 0.2564975321292877, "sampling/sampling_logp_difference/max": 1.3606362342834473, "sampling/sampling_logp_difference/mean": 0.03647936135530472, "step": 126, "step_time": 3.403859922895208 }, { "clip_ratio/high_max": 0.0022899978794157505, "clip_ratio/high_mean": 0.0022899978794157505, "clip_ratio/low_mean": 0.0016236664494499564, "clip_ratio/low_min": 0.0016236664494499564, "clip_ratio/region_mean": 0.003913664352148771, "completions/clipped_ratio": 0.0, "completions/max_length": 155.0, "completions/max_terminated_length": 155.0, "completions/mean_length": 53.21999740600586, "completions/mean_terminated_length": 53.21999740600586, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "entropy": 0.2477138340473175, "epoch": 0.3451086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.014389632269740105, "learning_rate": 5e-05, "loss": -0.0011505823349580169, "num_tokens": 5981421.0, "reward": 0.4540136754512787, "reward_std": 0.5024483799934387, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.025986328721046448, "rewards/length_penalty/std": 0.0136498948559165, "sampling/importance_sampling_ratio/max": 2.8656787872314453, "sampling/importance_sampling_ratio/mean": 0.9904687404632568, "sampling/importance_sampling_ratio/min": 0.3013729751110077, "sampling/sampling_logp_difference/max": 1.199406623840332, "sampling/sampling_logp_difference/mean": 0.03252456709742546, "step": 127, "step_time": 2.412539087701589 }, { "clip_ratio/high_max": 0.0012546269223093986, "clip_ratio/high_mean": 0.0012546269223093986, "clip_ratio/low_mean": 0.00281849279999733, "clip_ratio/low_min": 0.00281849279999733, "clip_ratio/region_mean": 0.004073119722306728, "completions/clipped_ratio": 0.0, "completions/max_length": 211.0, "completions/max_terminated_length": 211.0, "completions/mean_length": 36.040000915527344, "completions/mean_terminated_length": 36.040000915527344, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "entropy": 0.2539794921875, "epoch": 0.34782608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.01464737206697464, "learning_rate": 5e-05, "loss": 0.002901690546423197, "num_tokens": 5985543.0, "reward": 0.662402331829071, "reward_std": 0.4712248742580414, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.4712120592594147, "rewards/length_penalty/mean": -0.017597656697034836, "rewards/length_penalty/std": 0.013573279604315758, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9928796887397766, "sampling/importance_sampling_ratio/min": 0.210645854473114, "sampling/sampling_logp_difference/max": 1.5575768947601318, "sampling/sampling_logp_difference/mean": 0.04199233278632164, "step": 128, "step_time": 2.5844002419617027 }, { "clip_ratio/high_max": 0.002605846803635359, "clip_ratio/high_mean": 0.002605846803635359, "clip_ratio/low_mean": 0.0013825902715325356, "clip_ratio/low_min": 0.0013825902715325356, "clip_ratio/region_mean": 0.0039884370751678945, "completions/clipped_ratio": 0.0, "completions/max_length": 150.0, "completions/max_terminated_length": 150.0, "completions/mean_length": 45.81999969482422, "completions/mean_terminated_length": 45.81999969482422, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "entropy": 0.29810507595539093, "epoch": 0.35054347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.011710134334862232, "learning_rate": 5e-05, "loss": 0.003014075104147196, "num_tokens": 5990894.0, "reward": 0.39762693643569946, "reward_std": 0.5041302442550659, "rewards/correctness/mean": 0.41999998688697815, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.022373046725988388, "rewards/length_penalty/std": 0.013797881081700325, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9921415448188782, "sampling/importance_sampling_ratio/min": 0.32757124304771423, "sampling/sampling_logp_difference/max": 2.4522218704223633, "sampling/sampling_logp_difference/mean": 0.042723968625068665, "step": 129, "step_time": 2.299919289071113 }, { "clip_ratio/high_max": 0.00097189387306571, "clip_ratio/high_mean": 0.00097189387306571, "clip_ratio/low_mean": 0.0014031045138835907, "clip_ratio/low_min": 0.0014031045138835907, "clip_ratio/region_mean": 0.002374998386949301, "completions/clipped_ratio": 0.0, "completions/max_length": 188.0, "completions/max_terminated_length": 188.0, "completions/mean_length": 41.63999938964844, "completions/mean_terminated_length": 41.63999938964844, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "entropy": 0.2866079300642014, "epoch": 0.3532608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.014074258506298065, "learning_rate": 5e-05, "loss": 0.0013535830657929182, "num_tokens": 5995746.0, "reward": 0.6996679306030273, "reward_std": 0.45589467883110046, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.020332030951976776, "rewards/length_penalty/std": 0.01232099998742342, "sampling/importance_sampling_ratio/max": 2.545606851577759, "sampling/importance_sampling_ratio/mean": 0.9906572103500366, "sampling/importance_sampling_ratio/min": 0.1368415504693985, "sampling/sampling_logp_difference/max": 1.988931655883789, "sampling/sampling_logp_difference/mean": 0.04350081458687782, "step": 130, "step_time": 2.440223409794271 }, { "clip_ratio/high_max": 0.0022780835861340165, "clip_ratio/high_mean": 0.0022780835861340165, "clip_ratio/low_mean": 0.0007551305228844285, "clip_ratio/low_min": 0.0007551305228844285, "clip_ratio/region_mean": 0.003033214085735381, "completions/clipped_ratio": 0.0, "completions/max_length": 184.0, "completions/max_terminated_length": 184.0, "completions/mean_length": 48.5, "completions/mean_terminated_length": 48.5, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "entropy": 0.2793427348136902, "epoch": 0.35597826086956524, "frac_reward_zero_std": 0.0, "grad_norm": 0.010720326565206051, "learning_rate": 5e-05, "loss": 0.0011166753247380257, "num_tokens": 6001551.0, "reward": 0.4363183379173279, "reward_std": 0.5099425315856934, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.023681640625, "rewards/length_penalty/std": 0.016856711357831955, "sampling/importance_sampling_ratio/max": 2.0086348056793213, "sampling/importance_sampling_ratio/mean": 0.9880565404891968, "sampling/importance_sampling_ratio/min": 0.1856611669063568, "sampling/sampling_logp_difference/max": 1.6838319301605225, "sampling/sampling_logp_difference/mean": 0.03877510875463486, "step": 131, "step_time": 2.758909401949495 }, { "clip_ratio/high_max": 0.0014819981530308723, "clip_ratio/high_mean": 0.0014819981530308723, "clip_ratio/low_mean": 0.0006134969182312489, "clip_ratio/low_min": 0.0006134969182312489, "clip_ratio/region_mean": 0.0020954950712621214, "completions/clipped_ratio": 0.0, "completions/max_length": 96.0, "completions/max_terminated_length": 96.0, "completions/mean_length": 37.15999984741211, "completions/mean_terminated_length": 37.15999984741211, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "entropy": 0.22479414641857148, "epoch": 0.358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.00991136860102415, "learning_rate": 5e-05, "loss": 0.0004250574274919927, "num_tokens": 6005239.0, "reward": 0.44185546040534973, "reward_std": 0.5011681318283081, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.018144531175494194, "rewards/length_penalty/std": 0.010862166061997414, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.991742730140686, "sampling/importance_sampling_ratio/min": 0.3487538695335388, "sampling/sampling_logp_difference/max": 1.4278502464294434, "sampling/sampling_logp_difference/mean": 0.03862390294671059, "step": 132, "step_time": 1.9299116518814117 }, { "clip_ratio/high_max": 0.001252327300608158, "clip_ratio/high_mean": 0.001252327300608158, "clip_ratio/low_mean": 0.0018417619168758393, "clip_ratio/low_min": 0.0018417619168758393, "clip_ratio/region_mean": 0.0030940892174839973, "completions/clipped_ratio": 0.0, "completions/max_length": 88.0, "completions/max_terminated_length": 88.0, "completions/mean_length": 33.15999984741211, "completions/mean_terminated_length": 33.15999984741211, "completions/min_length": 18.0, "completions/min_terminated_length": 18.0, "entropy": 0.2817714989185333, "epoch": 0.36141304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.008105136454105377, "learning_rate": 5e-05, "loss": -0.0009989547543227673, "num_tokens": 6009587.0, "reward": 0.42380857467651367, "reward_std": 0.5010993480682373, "rewards/correctness/mean": 0.4399999976158142, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.016191406175494194, "rewards/length_penalty/std": 0.007762114051729441, "sampling/importance_sampling_ratio/max": 1.943861961364746, "sampling/importance_sampling_ratio/mean": 0.9887738227844238, "sampling/importance_sampling_ratio/min": 0.2915809452533722, "sampling/sampling_logp_difference/max": 1.2324376106262207, "sampling/sampling_logp_difference/mean": 0.039197519421577454, "step": 133, "step_time": 1.915332199074328 }, { "clip_ratio/high_max": 0.0008230452425777912, "clip_ratio/high_mean": 0.0008230452425777912, "clip_ratio/low_mean": 0.0007968127727508544, "clip_ratio/low_min": 0.0007968127727508544, "clip_ratio/region_mean": 0.0016198580153286457, "completions/clipped_ratio": 0.0, "completions/max_length": 58.0, "completions/max_terminated_length": 58.0, "completions/mean_length": 25.079999923706055, "completions/mean_terminated_length": 25.079999923706055, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 0.19418410062789918, "epoch": 0.3641304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.010841830633580685, "learning_rate": 5e-05, "loss": -0.0005958230467513204, "num_tokens": 6013531.0, "reward": 0.5877538919448853, "reward_std": 0.4935136139392853, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.012246093712747097, "rewards/length_penalty/std": 0.004991067107766867, "sampling/importance_sampling_ratio/max": 2.5555429458618164, "sampling/importance_sampling_ratio/mean": 0.9979280829429626, "sampling/importance_sampling_ratio/min": 0.10886480659246445, "sampling/sampling_logp_difference/max": 2.217648506164551, "sampling/sampling_logp_difference/mean": 0.035169076174497604, "step": 134, "step_time": 1.825435406761244 }, { "clip_ratio/high_max": 0.0028490145690739156, "clip_ratio/high_mean": 0.0028490145690739156, "clip_ratio/low_mean": 0.001599618257023394, "clip_ratio/low_min": 0.001599618257023394, "clip_ratio/region_mean": 0.004448632849380374, "completions/clipped_ratio": 0.0, "completions/max_length": 250.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 33.68000030517578, "completions/mean_terminated_length": 33.68000030517578, "completions/min_length": 18.0, "completions/min_terminated_length": 18.0, "entropy": 0.2851863861083984, "epoch": 0.36684782608695654, "frac_reward_zero_std": 0.0, "grad_norm": 0.008571211248636246, "learning_rate": 5e-05, "loss": 0.002348761772736907, "num_tokens": 6019395.0, "reward": 0.12355468422174454, "reward_std": 0.35180458426475525, "rewards/correctness/mean": 0.14000000059604645, "rewards/correctness/std": 0.3505098223686218, "rewards/length_penalty/mean": -0.016445312649011612, "rewards/length_penalty/std": 0.017781900241971016, "sampling/importance_sampling_ratio/max": 2.548715114593506, "sampling/importance_sampling_ratio/mean": 0.9878785610198975, "sampling/importance_sampling_ratio/min": 0.14688512682914734, "sampling/sampling_logp_difference/max": 1.9181044101715088, "sampling/sampling_logp_difference/mean": 0.04530085623264313, "step": 135, "step_time": 3.565745212836191 }, { "clip_ratio/high_max": 0.002418273827061057, "clip_ratio/high_mean": 0.002418273827061057, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002418273827061057, "completions/clipped_ratio": 0.0, "completions/max_length": 65.0, "completions/max_terminated_length": 65.0, "completions/mean_length": 33.36000061035156, "completions/mean_terminated_length": 33.36000061035156, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "entropy": 0.2012420564889908, "epoch": 0.3695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.012487429194152355, "learning_rate": 5e-05, "loss": 0.0007071903673931956, "num_tokens": 6023143.0, "reward": 0.6437109112739563, "reward_std": 0.47799545526504517, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.47851812839508057, "rewards/length_penalty/mean": -0.016289062798023224, "rewards/length_penalty/std": 0.006368517410010099, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.99516761302948, "sampling/importance_sampling_ratio/min": 0.3091590404510498, "sampling/sampling_logp_difference/max": 1.1738994121551514, "sampling/sampling_logp_difference/mean": 0.035100940614938736, "step": 136, "step_time": 1.8060238698963076 }, { "clip_ratio/high_max": 0.0006153846159577369, "clip_ratio/high_mean": 0.0006153846159577369, "clip_ratio/low_mean": 0.0006153846159577369, "clip_ratio/low_min": 0.0006153846159577369, "clip_ratio/region_mean": 0.0012307692319154739, "completions/clipped_ratio": 0.0, "completions/max_length": 70.0, "completions/max_terminated_length": 70.0, "completions/mean_length": 41.459999084472656, "completions/mean_terminated_length": 41.459999084472656, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "entropy": 0.17081710696220398, "epoch": 0.37228260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.00821436196565628, "learning_rate": 5e-05, "loss": 0.0015817350940778852, "num_tokens": 6028076.0, "reward": 0.5397558212280273, "reward_std": 0.5067362785339355, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.020244140177965164, "rewards/length_penalty/std": 0.009693717584013939, "sampling/importance_sampling_ratio/max": 1.7419631481170654, "sampling/importance_sampling_ratio/mean": 0.9927988648414612, "sampling/importance_sampling_ratio/min": 0.35504549741744995, "sampling/sampling_logp_difference/max": 1.0355093479156494, "sampling/sampling_logp_difference/mean": 0.02509792521595955, "step": 137, "step_time": 1.8471284741535783 }, { "clip_ratio/high_max": 0.0036086251493543386, "clip_ratio/high_mean": 0.0036086251493543386, "clip_ratio/low_mean": 0.0004056795034557581, "clip_ratio/low_min": 0.0004056795034557581, "clip_ratio/region_mean": 0.004014304652810097, "completions/clipped_ratio": 0.0, "completions/max_length": 188.0, "completions/max_terminated_length": 188.0, "completions/mean_length": 38.63999938964844, "completions/mean_terminated_length": 38.63999938964844, "completions/min_length": 18.0, "completions/min_terminated_length": 18.0, "entropy": 0.2256270557641983, "epoch": 0.375, "frac_reward_zero_std": 0.0, "grad_norm": 0.008576368913054466, "learning_rate": 5e-05, "loss": 0.003299054689705372, "num_tokens": 6033038.0, "reward": 0.5211328268051147, "reward_std": 0.5057582855224609, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.018867187201976776, "rewards/length_penalty/std": 0.013564602471888065, "sampling/importance_sampling_ratio/max": 1.6007344722747803, "sampling/importance_sampling_ratio/mean": 0.985950231552124, "sampling/importance_sampling_ratio/min": 0.07781390100717545, "sampling/sampling_logp_difference/max": 2.5534350872039795, "sampling/sampling_logp_difference/mean": 0.03910902887582779, "step": 138, "step_time": 2.515745670069009 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00024968788493424655, "clip_ratio/low_min": 0.00024968788493424655, "clip_ratio/region_mean": 0.00024968788493424655, "completions/clipped_ratio": 0.0, "completions/max_length": 445.0, "completions/max_terminated_length": 445.0, "completions/mean_length": 37.70000076293945, "completions/mean_terminated_length": 37.70000076293945, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "entropy": 0.2630132257938385, "epoch": 0.37771739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.011270148679614067, "learning_rate": 5e-05, "loss": 0.0028373789973556995, "num_tokens": 6037703.0, "reward": 0.18159179389476776, "reward_std": 0.4050198197364807, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.01840820349752903, "rewards/length_penalty/std": 0.030904652550816536, "sampling/importance_sampling_ratio/max": 1.4473143815994263, "sampling/importance_sampling_ratio/mean": 0.9855800271034241, "sampling/importance_sampling_ratio/min": 0.45448043942451477, "sampling/sampling_logp_difference/max": 0.7886004447937012, "sampling/sampling_logp_difference/mean": 0.02931682951748371, "step": 139, "step_time": 4.710732836974785 }, { "clip_ratio/high_max": 0.0039062283933162687, "clip_ratio/high_mean": 0.0039062283933162687, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0039062283933162687, "completions/clipped_ratio": 0.0, "completions/max_length": 104.0, "completions/max_terminated_length": 104.0, "completions/mean_length": 34.459999084472656, "completions/mean_terminated_length": 34.459999084472656, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 0.28029139935970304, "epoch": 0.3804347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.007939372211694717, "learning_rate": 5e-05, "loss": 0.001906823250465095, "num_tokens": 6043066.0, "reward": 0.48317381739616394, "reward_std": 0.5078606009483337, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.016826171427965164, "rewards/length_penalty/std": 0.008946106769144535, "sampling/importance_sampling_ratio/max": 2.1778173446655273, "sampling/importance_sampling_ratio/mean": 0.9844793081283569, "sampling/importance_sampling_ratio/min": 0.2821851074695587, "sampling/sampling_logp_difference/max": 1.2651920318603516, "sampling/sampling_logp_difference/mean": 0.039756450802087784, "step": 140, "step_time": 2.190974553814158 }, { "clip_ratio/high_max": 0.0021777387242764234, "clip_ratio/high_mean": 0.0021777387242764234, "clip_ratio/low_mean": 0.000526315812021494, "clip_ratio/low_min": 0.000526315812021494, "clip_ratio/region_mean": 0.00270405444316566, "completions/clipped_ratio": 0.0, "completions/max_length": 100.0, "completions/max_terminated_length": 100.0, "completions/mean_length": 44.41999816894531, "completions/mean_terminated_length": 44.41999816894531, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.22660049200057983, "epoch": 0.38315217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.010583036579191685, "learning_rate": 5e-05, "loss": 0.0017442009411752224, "num_tokens": 6048017.0, "reward": 0.4783105254173279, "reward_std": 0.5048611164093018, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.02168945223093033, "rewards/length_penalty/std": 0.011465261690318584, "sampling/importance_sampling_ratio/max": 1.4931265115737915, "sampling/importance_sampling_ratio/mean": 0.9867258071899414, "sampling/importance_sampling_ratio/min": 0.09107369929552078, "sampling/sampling_logp_difference/max": 2.3960862159729004, "sampling/sampling_logp_difference/mean": 0.03287019953131676, "step": 141, "step_time": 2.040590178454295 }, { "clip_ratio/high_max": 0.002919093193486333, "clip_ratio/high_mean": 0.002919093193486333, "clip_ratio/low_mean": 0.0005555555690079927, "clip_ratio/low_min": 0.0005555555690079927, "clip_ratio/region_mean": 0.003474648762494326, "completions/clipped_ratio": 0.0, "completions/max_length": 76.0, "completions/max_terminated_length": 76.0, "completions/mean_length": 29.8799991607666, "completions/mean_terminated_length": 29.8799991607666, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.2625975370407104, "epoch": 0.3858695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.012548496946692467, "learning_rate": 5e-05, "loss": 0.0033927010372281075, "num_tokens": 6052301.0, "reward": 0.4254101514816284, "reward_std": 0.5026375651359558, "rewards/correctness/mean": 0.4399999976158142, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.014589843340218067, "rewards/length_penalty/std": 0.008529257029294968, "sampling/importance_sampling_ratio/max": 2.683242082595825, "sampling/importance_sampling_ratio/mean": 0.9934856295585632, "sampling/importance_sampling_ratio/min": 0.07049904763698578, "sampling/sampling_logp_difference/max": 2.652156114578247, "sampling/sampling_logp_difference/mean": 0.04649779945611954, "step": 142, "step_time": 1.7738669028040022 }, { "clip_ratio/high_max": 0.0008064515888690949, "clip_ratio/high_mean": 0.0008064515888690949, "clip_ratio/low_mean": 0.00048076924867928027, "clip_ratio/low_min": 0.00048076924867928027, "clip_ratio/region_mean": 0.0012872208375483752, "completions/clipped_ratio": 0.0, "completions/max_length": 92.0, "completions/max_terminated_length": 92.0, "completions/mean_length": 35.23999786376953, "completions/mean_terminated_length": 35.23999786376953, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.18691827356815338, "epoch": 0.38858695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.009847329929471016, "learning_rate": 5e-05, "loss": 0.0008599873399361968, "num_tokens": 6056403.0, "reward": 0.44279295206069946, "reward_std": 0.5066936612129211, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.017207032069563866, "rewards/length_penalty/std": 0.01047922857105732, "sampling/importance_sampling_ratio/max": 2.7992422580718994, "sampling/importance_sampling_ratio/mean": 0.9902405142784119, "sampling/importance_sampling_ratio/min": 0.17679700255393982, "sampling/sampling_logp_difference/max": 1.732753038406372, "sampling/sampling_logp_difference/mean": 0.034317996352910995, "step": 143, "step_time": 2.020391206955537 }, { "clip_ratio/high_max": 0.000803212821483612, "clip_ratio/high_mean": 0.000803212821483612, "clip_ratio/low_mean": 0.0014033276587724687, "clip_ratio/low_min": 0.0014033276587724687, "clip_ratio/region_mean": 0.0022065404802560805, "completions/clipped_ratio": 0.0, "completions/max_length": 92.0, "completions/max_terminated_length": 92.0, "completions/mean_length": 28.059999465942383, "completions/mean_terminated_length": 28.059999465942383, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.1982872635126114, "epoch": 0.391304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.012820910662412643, "learning_rate": 5e-05, "loss": 0.0011645274935290217, "num_tokens": 6060856.0, "reward": 0.24629881978034973, "reward_std": 0.444132924079895, "rewards/correctness/mean": 0.25999999046325684, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.013701171614229679, "rewards/length_penalty/std": 0.008656531572341919, "sampling/importance_sampling_ratio/max": 2.300398588180542, "sampling/importance_sampling_ratio/mean": 0.9945727586746216, "sampling/importance_sampling_ratio/min": 0.28775298595428467, "sampling/sampling_logp_difference/max": 1.2456529140472412, "sampling/sampling_logp_difference/mean": 0.03747813031077385, "step": 144, "step_time": 1.8891234768088907 }, { "clip_ratio/high_max": 0.0028685005381703377, "clip_ratio/high_mean": 0.0028685005381703377, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028685005381703377, "completions/clipped_ratio": 0.0, "completions/max_length": 35.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 20.760000228881836, "completions/mean_terminated_length": 20.760000228881836, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.11449335515499115, "epoch": 0.39402173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.022868067026138306, "learning_rate": 5e-05, "loss": 0.001642507268115878, "num_tokens": 6064224.0, "reward": 0.46986326575279236, "reward_std": 0.5068603754043579, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.010136718861758709, "rewards/length_penalty/std": 0.004064429551362991, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9941112399101257, "sampling/importance_sampling_ratio/min": 0.18363036215305328, "sampling/sampling_logp_difference/max": 1.6948304176330566, "sampling/sampling_logp_difference/mean": 0.0298063512891531, "step": 145, "step_time": 1.5340910970699042 }, { "clip_ratio/high_max": 0.005621016025543213, "clip_ratio/high_mean": 0.005621016025543213, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005621016025543213, "completions/clipped_ratio": 0.0, "completions/max_length": 37.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 14.34000015258789, "completions/mean_terminated_length": 14.34000015258789, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.15054220706224442, "epoch": 0.3967391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.012355758808553219, "learning_rate": 5e-05, "loss": 0.0012912503443658352, "num_tokens": 6067881.0, "reward": 0.5529980063438416, "reward_std": 0.5011084079742432, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.007001953199505806, "rewards/length_penalty/std": 0.0030906975734978914, "sampling/importance_sampling_ratio/max": 2.6878232955932617, "sampling/importance_sampling_ratio/mean": 0.9853367805480957, "sampling/importance_sampling_ratio/min": 0.16923557221889496, "sampling/sampling_logp_difference/max": 1.7764636278152466, "sampling/sampling_logp_difference/mean": 0.05857422575354576, "step": 146, "step_time": 1.4935915581882 }, { "clip_ratio/high_max": 0.004120059683918953, "clip_ratio/high_mean": 0.004120059683918953, "clip_ratio/low_mean": 0.001342281885445118, "clip_ratio/low_min": 0.001342281885445118, "clip_ratio/region_mean": 0.0054623415693640705, "completions/clipped_ratio": 0.0, "completions/max_length": 29.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 14.15999984741211, "completions/mean_terminated_length": 14.15999984741211, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.09627678245306015, "epoch": 0.39945652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.048468802124261856, "learning_rate": 5e-05, "loss": 0.0005410637240856886, "num_tokens": 6070889.0, "reward": 0.33308592438697815, "reward_std": 0.4778694212436676, "rewards/correctness/mean": 0.3400000035762787, "rewards/correctness/std": 0.4785180985927582, "rewards/length_penalty/mean": -0.006914062425494194, "rewards/length_penalty/std": 0.0017294606659561396, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9871498346328735, "sampling/importance_sampling_ratio/min": 0.21893155574798584, "sampling/sampling_logp_difference/max": 1.5690662860870361, "sampling/sampling_logp_difference/mean": 0.03849093243479729, "step": 147, "step_time": 1.9382816620636731 }, { "clip_ratio/high_max": 0.001785714365541935, "clip_ratio/high_mean": 0.001785714365541935, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001785714365541935, "completions/clipped_ratio": 0.0, "completions/max_length": 14.0, "completions/max_terminated_length": 14.0, "completions/mean_length": 11.219999313354492, "completions/mean_terminated_length": 11.219999313354492, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.09468620419502258, "epoch": 0.40217391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.013497934676706791, "learning_rate": 5e-05, "loss": 0.0001785533968359232, "num_tokens": 6074360.0, "reward": 0.1945214867591858, "reward_std": 0.4041154980659485, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.0054785157553851604, "rewards/length_penalty/std": 0.0004101894737686962, "sampling/importance_sampling_ratio/max": 1.3334838151931763, "sampling/importance_sampling_ratio/mean": 0.9918619394302368, "sampling/importance_sampling_ratio/min": 0.1652003973722458, "sampling/sampling_logp_difference/max": 1.800595998764038, "sampling/sampling_logp_difference/mean": 0.02222876064479351, "step": 148, "step_time": 1.357269709231332 }, { "clip_ratio/high_max": 0.0011363636702299118, "clip_ratio/high_mean": 0.0011363636702299118, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011363636702299118, "completions/clipped_ratio": 0.0, "completions/max_length": 36.0, "completions/max_terminated_length": 36.0, "completions/mean_length": 14.019999504089355, "completions/mean_terminated_length": 14.019999504089355, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 0.1659936398267746, "epoch": 0.4048913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.008294458501040936, "learning_rate": 5e-05, "loss": 0.0007371972315013409, "num_tokens": 6078031.0, "reward": 0.1731542944908142, "reward_std": 0.38879117369651794, "rewards/correctness/mean": 0.18000000715255737, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.006845703348517418, "rewards/length_penalty/std": 0.002153346547856927, "sampling/importance_sampling_ratio/max": 1.6676440238952637, "sampling/importance_sampling_ratio/mean": 0.9931860566139221, "sampling/importance_sampling_ratio/min": 0.24514369666576385, "sampling/sampling_logp_difference/max": 1.4059107303619385, "sampling/sampling_logp_difference/mean": 0.02321179211139679, "step": 149, "step_time": 1.5234273611567914 }, { "clip_ratio/high_max": 0.0015037594363093375, "clip_ratio/high_mean": 0.0015037594363093375, "clip_ratio/low_mean": 0.0015037594363093375, "clip_ratio/low_min": 0.0015037594363093375, "clip_ratio/region_mean": 0.003007518872618675, "completions/clipped_ratio": 0.0, "completions/max_length": 30.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 11.5, "completions/mean_terminated_length": 11.5, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.030891793966293334, "epoch": 0.4076086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.012591220438480377, "learning_rate": 5e-05, "loss": -0.00041141733527183533, "num_tokens": 6080366.0, "reward": 0.5543847680091858, "reward_std": 0.5016669034957886, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.005615234375, "rewards/length_penalty/std": 0.0014098342508077621, "sampling/importance_sampling_ratio/max": 1.1196001768112183, "sampling/importance_sampling_ratio/mean": 0.9954295754432678, "sampling/importance_sampling_ratio/min": 0.48469099402427673, "sampling/sampling_logp_difference/max": 0.7242437601089478, "sampling/sampling_logp_difference/mean": 0.009189309552311897, "step": 150, "step_time": 1.4204991972073913 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 20.0, "completions/max_terminated_length": 20.0, "completions/mean_length": 11.420000076293945, "completions/mean_terminated_length": 11.420000076293945, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.04515114948153496, "epoch": 0.41032608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.01136438362300396, "learning_rate": 5e-05, "loss": 0.0006140059558674693, "num_tokens": 6083907.0, "reward": 0.39442381262779236, "reward_std": 0.49504169821739197, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.005576171912252903, "rewards/length_penalty/std": 0.0009466582559980452, "sampling/importance_sampling_ratio/max": 1.0522286891937256, "sampling/importance_sampling_ratio/mean": 0.9954703450202942, "sampling/importance_sampling_ratio/min": 0.7883068919181824, "sampling/sampling_logp_difference/max": 0.2378678321838379, "sampling/sampling_logp_difference/mean": 0.0067811706103384495, "step": 151, "step_time": 1.4778102801647037 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 24.0, "completions/max_terminated_length": 24.0, "completions/mean_length": 11.479999542236328, "completions/mean_terminated_length": 11.479999542236328, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.05021545886993408, "epoch": 0.41304347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.013961839489638805, "learning_rate": 5e-05, "loss": 0.0005877700168639421, "num_tokens": 6087661.0, "reward": 0.7343944907188416, "reward_std": 0.4431508183479309, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.005605468526482582, "rewards/length_penalty/std": 0.0010497525800019503, "sampling/importance_sampling_ratio/max": 1.712555170059204, "sampling/importance_sampling_ratio/mean": 1.0007383823394775, "sampling/importance_sampling_ratio/min": 0.49543195962905884, "sampling/sampling_logp_difference/max": 0.7023252248764038, "sampling/sampling_logp_difference/mean": 0.009280516766011715, "step": 152, "step_time": 1.4290544828400016 }, { "clip_ratio/high_max": 0.0017699114978313446, "clip_ratio/high_mean": 0.0017699114978313446, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017699114978313446, "completions/clipped_ratio": 0.0, "completions/max_length": 15.0, "completions/max_terminated_length": 15.0, "completions/mean_length": 11.5, "completions/mean_terminated_length": 11.5, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.19660144448280334, "epoch": 0.4157608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.008950591087341309, "learning_rate": 5e-05, "loss": 0.00016954117745626718, "num_tokens": 6091756.0, "reward": 0.11438476294279099, "reward_std": 0.3285348415374756, "rewards/correctness/mean": 0.11999999731779099, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.005615234375, "rewards/length_penalty/std": 0.0008209125371649861, "sampling/importance_sampling_ratio/max": 1.3838289976119995, "sampling/importance_sampling_ratio/mean": 0.9945533275604248, "sampling/importance_sampling_ratio/min": 0.6776381731033325, "sampling/sampling_logp_difference/max": 0.3891417980194092, "sampling/sampling_logp_difference/mean": 0.017728928476572037, "step": 153, "step_time": 1.3570173350162804 }, { "clip_ratio/high_max": 0.001652892492711544, "clip_ratio/high_mean": 0.001652892492711544, "clip_ratio/low_mean": 0.0015267175622284412, "clip_ratio/low_min": 0.0015267175622284412, "clip_ratio/region_mean": 0.003179610054939985, "completions/clipped_ratio": 0.0, "completions/max_length": 25.0, "completions/max_terminated_length": 25.0, "completions/mean_length": 13.399999618530273, "completions/mean_terminated_length": 13.399999618530273, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.10291488319635392, "epoch": 0.41847826086956524, "frac_reward_zero_std": 0.0, "grad_norm": 0.028912127017974854, "learning_rate": 5e-05, "loss": 6.605967064388096e-05, "num_tokens": 6096186.0, "reward": 0.33345702290534973, "reward_std": 0.4795200228691101, "rewards/correctness/mean": 0.3400000035762787, "rewards/correctness/std": 0.47851812839508057, "rewards/length_penalty/mean": -0.006542968563735485, "rewards/length_penalty/std": 0.002685320330783725, "sampling/importance_sampling_ratio/max": 1.365503191947937, "sampling/importance_sampling_ratio/mean": 0.9983199238777161, "sampling/importance_sampling_ratio/min": 0.7811017632484436, "sampling/sampling_logp_difference/max": 0.3115229606628418, "sampling/sampling_logp_difference/mean": 0.010536031797528267, "step": 154, "step_time": 1.4563604779541492 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13.0, "completions/max_terminated_length": 13.0, "completions/mean_length": 11.65999984741211, "completions/mean_terminated_length": 11.65999984741211, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.09399411082267761, "epoch": 0.421195652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.008154707960784435, "learning_rate": 5e-05, "loss": 0.0002479680988471955, "num_tokens": 6099799.0, "reward": 0.1943066418170929, "reward_std": 0.4042239189147949, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.005693359300494194, "rewards/length_penalty/std": 0.0003897520946338773, "sampling/importance_sampling_ratio/max": 1.1896085739135742, "sampling/importance_sampling_ratio/mean": 0.994530200958252, "sampling/importance_sampling_ratio/min": 0.6909244060516357, "sampling/sampling_logp_difference/max": 0.3697248697280884, "sampling/sampling_logp_difference/mean": 0.009416679851710796, "step": 155, "step_time": 1.3759884182363749 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0019417475908994676, "clip_ratio/low_min": 0.0019417475908994676, "clip_ratio/region_mean": 0.0019417475908994676, "completions/clipped_ratio": 0.0, "completions/max_length": 11.0, "completions/max_terminated_length": 11.0, "completions/mean_length": 10.279999732971191, "completions/mean_terminated_length": 10.279999732971191, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.08822216242551803, "epoch": 0.42391304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.010327600874006748, "learning_rate": 5e-05, "loss": 3.508696681819856e-05, "num_tokens": 6103873.0, "reward": 0.35498046875, "reward_std": 0.4849768579006195, "rewards/correctness/mean": 0.36000001430511475, "rewards/correctness/std": 0.4848732352256775, "rewards/length_penalty/mean": -0.0050195311196148396, "rewards/length_penalty/std": 0.00022146356059238315, "sampling/importance_sampling_ratio/max": 1.2830084562301636, "sampling/importance_sampling_ratio/mean": 0.994913637638092, "sampling/importance_sampling_ratio/min": 0.7795566916465759, "sampling/sampling_logp_difference/max": 0.2492077350616455, "sampling/sampling_logp_difference/mean": 0.008491582237184048, "step": 156, "step_time": 1.348271707072854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14.0, "completions/max_terminated_length": 14.0, "completions/mean_length": 11.34000015258789, "completions/mean_terminated_length": 11.34000015258789, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.04637726470828056, "epoch": 0.4266304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.00861538015305996, "learning_rate": 5e-05, "loss": 4.4768385123461485e-05, "num_tokens": 6107120.0, "reward": 0.21446289122104645, "reward_std": 0.4187796413898468, "rewards/correctness/mean": 0.2199999988079071, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.005537109449505806, "rewards/length_penalty/std": 0.0007155189523473382, "sampling/importance_sampling_ratio/max": 1.0811917781829834, "sampling/importance_sampling_ratio/mean": 0.9970145225524902, "sampling/importance_sampling_ratio/min": 0.6817883253097534, "sampling/sampling_logp_difference/max": 0.38303613662719727, "sampling/sampling_logp_difference/mean": 0.00469635147601366, "step": 157, "step_time": 1.4151484090834856 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.003883495181798935, "clip_ratio/low_min": 0.003883495181798935, "clip_ratio/region_mean": 0.003883495181798935, "completions/clipped_ratio": 0.0, "completions/max_length": 12.0, "completions/max_terminated_length": 12.0, "completions/mean_length": 10.34000015258789, "completions/mean_terminated_length": 10.34000015258789, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.08666332289576531, "epoch": 0.42934782608695654, "frac_reward_zero_std": 0.0, "grad_norm": 0.020958509296178818, "learning_rate": 5e-05, "loss": 8.026038995012641e-05, "num_tokens": 6111047.0, "reward": 0.27495115995407104, "reward_std": 0.4536620080471039, "rewards/correctness/mean": 0.2800000011920929, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.005048828199505806, "rewards/length_penalty/std": 0.0002536224783398211, "sampling/importance_sampling_ratio/max": 1.541198492050171, "sampling/importance_sampling_ratio/mean": 0.9994328618049622, "sampling/importance_sampling_ratio/min": 0.8014639019966125, "sampling/sampling_logp_difference/max": 0.4325603246688843, "sampling/sampling_logp_difference/mean": 0.0076075028628110886, "step": 158, "step_time": 1.3667166538070887 }, { "clip_ratio/high_max": 0.003166061732918024, "clip_ratio/high_mean": 0.003166061732918024, "clip_ratio/low_mean": 0.001449275389313698, "clip_ratio/low_min": 0.001449275389313698, "clip_ratio/region_mean": 0.004615337122231722, "completions/clipped_ratio": 0.0, "completions/max_length": 21.0, "completions/max_terminated_length": 21.0, "completions/mean_length": 13.09999942779541, "completions/mean_terminated_length": 13.09999942779541, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.10265765637159348, "epoch": 0.4320652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.03846745565533638, "learning_rate": 5e-05, "loss": 0.0003612680302467197, "num_tokens": 6114972.0, "reward": 0.11360350996255875, "reward_std": 0.328829288482666, "rewards/correctness/mean": 0.11999999731779099, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.006396484561264515, "rewards/length_penalty/std": 0.0017036376520991325, "sampling/importance_sampling_ratio/max": 1.473070740699768, "sampling/importance_sampling_ratio/mean": 0.9970410466194153, "sampling/importance_sampling_ratio/min": 0.6239770650863647, "sampling/sampling_logp_difference/max": 0.4716416597366333, "sampling/sampling_logp_difference/mean": 0.013403923250734806, "step": 159, "step_time": 1.4047386380843818 }, { "clip_ratio/high_max": 0.0016949152573943137, "clip_ratio/high_mean": 0.0016949152573943137, "clip_ratio/low_mean": 0.0016949152573943137, "clip_ratio/low_min": 0.0016949152573943137, "clip_ratio/region_mean": 0.0033898305147886275, "completions/clipped_ratio": 0.0, "completions/max_length": 18.0, "completions/max_terminated_length": 18.0, "completions/mean_length": 11.479999542236328, "completions/mean_terminated_length": 11.479999542236328, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.15147124826908112, "epoch": 0.43478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.010108926333487034, "learning_rate": 5e-05, "loss": 0.0001797530276235193, "num_tokens": 6119096.0, "reward": 0.11439453065395355, "reward_std": 0.3284417986869812, "rewards/correctness/mean": 0.11999999731779099, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.005605468526482582, "rewards/length_penalty/std": 0.0013170115416869521, "sampling/importance_sampling_ratio/max": 1.2050960063934326, "sampling/importance_sampling_ratio/mean": 0.994880735874176, "sampling/importance_sampling_ratio/min": 0.15859197080135345, "sampling/sampling_logp_difference/max": 1.8414206504821777, "sampling/sampling_logp_difference/mean": 0.015831688418984413, "step": 160, "step_time": 1.8692998373880982 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11.0, "completions/max_terminated_length": 11.0, "completions/mean_length": 10.399999618530273, "completions/mean_terminated_length": 10.399999618530273, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.09015260264277458, "epoch": 0.4375, "frac_reward_zero_std": 0.0, "grad_norm": 0.006907617673277855, "learning_rate": 5e-05, "loss": 1.201988197863102e-06, "num_tokens": 6123046.0, "reward": 0.4549218714237213, "reward_std": 0.5036197900772095, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.005078124813735485, "rewards/length_penalty/std": 0.00024163654597941786, "sampling/importance_sampling_ratio/max": 1.218556523323059, "sampling/importance_sampling_ratio/mean": 0.9941509366035461, "sampling/importance_sampling_ratio/min": 0.6940081119537354, "sampling/sampling_logp_difference/max": 0.3652716875076294, "sampling/sampling_logp_difference/mean": 0.008760688826441765, "step": 161, "step_time": 1.367599799996242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 24.0, "completions/max_terminated_length": 24.0, "completions/mean_length": 13.4399995803833, "completions/mean_terminated_length": 13.4399995803833, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.08651992529630662, "epoch": 0.44021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.010480268858373165, "learning_rate": 5e-05, "loss": 0.00023603468434885144, "num_tokens": 6126308.0, "reward": -0.006562499795109034, "reward_std": 0.0025384491309523582, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.006562499795109034, "rewards/length_penalty/std": 0.0025384491309523582, "sampling/importance_sampling_ratio/max": 1.6141817569732666, "sampling/importance_sampling_ratio/mean": 0.9978366494178772, "sampling/importance_sampling_ratio/min": 0.7670055031776428, "sampling/sampling_logp_difference/max": 0.47882819175720215, "sampling/sampling_logp_difference/mean": 0.008288954384624958, "step": 162, "step_time": 1.4457339339423925 }, { "clip_ratio/high_max": 0.0017391303554177284, "clip_ratio/high_mean": 0.0017391303554177284, "clip_ratio/low_mean": 0.001652892492711544, "clip_ratio/low_min": 0.001652892492711544, "clip_ratio/region_mean": 0.0033920228481292723, "completions/clipped_ratio": 0.0, "completions/max_length": 14.0, "completions/max_terminated_length": 14.0, "completions/mean_length": 11.800000190734863, "completions/mean_terminated_length": 11.800000190734863, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 0.06177229434251785, "epoch": 0.4429347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.0064376164227724075, "learning_rate": 5e-05, "loss": -4.822401388082653e-05, "num_tokens": 6129608.0, "reward": 0.03423827886581421, "reward_std": 0.19797970354557037, "rewards/correctness/mean": 0.03999999910593033, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.0057617188431322575, "rewards/length_penalty/std": 0.000575210084207356, "sampling/importance_sampling_ratio/max": 1.7795566320419312, "sampling/importance_sampling_ratio/mean": 0.9969757199287415, "sampling/importance_sampling_ratio/min": 0.5768961906433105, "sampling/sampling_logp_difference/max": 0.576364278793335, "sampling/sampling_logp_difference/mean": 0.010751276277005672, "step": 163, "step_time": 1.3526620748452842 }, { "clip_ratio/high_max": 0.0032133178785443304, "clip_ratio/high_mean": 0.0032133178785443304, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032133178785443304, "completions/clipped_ratio": 0.0, "completions/max_length": 21.0, "completions/max_terminated_length": 21.0, "completions/mean_length": 12.779999732971191, "completions/mean_terminated_length": 12.779999732971191, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.11402323693037034, "epoch": 0.44565217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.014964309521019459, "learning_rate": 5e-05, "loss": 0.00010747273336164653, "num_tokens": 6133107.0, "reward": 0.33375975489616394, "reward_std": 0.47908881306648254, "rewards/correctness/mean": 0.3400000035762787, "rewards/correctness/std": 0.4785180985927582, "rewards/length_penalty/mean": -0.0062402342446148396, "rewards/length_penalty/std": 0.0017928117886185646, "sampling/importance_sampling_ratio/max": 1.1915403604507446, "sampling/importance_sampling_ratio/mean": 0.9944223761558533, "sampling/importance_sampling_ratio/min": 0.6367499828338623, "sampling/sampling_logp_difference/max": 0.4513782262802124, "sampling/sampling_logp_difference/mean": 0.01162565965205431, "step": 164, "step_time": 1.4370824880897999 }, { "clip_ratio/high_max": 0.004844521544873715, "clip_ratio/high_mean": 0.004844521544873715, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004844521544873715, "completions/clipped_ratio": 0.0, "completions/max_length": 24.0, "completions/max_terminated_length": 24.0, "completions/mean_length": 12.859999656677246, "completions/mean_terminated_length": 12.859999656677246, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.09507749825716019, "epoch": 0.4483695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.01483475137501955, "learning_rate": 5e-05, "loss": 0.0004417179152369499, "num_tokens": 6137120.0, "reward": 0.0937207043170929, "reward_std": 0.30319303274154663, "rewards/correctness/mean": 0.10000000149011612, "rewards/correctness/std": 0.30304574966430664, "rewards/length_penalty/mean": -0.006279296707361937, "rewards/length_penalty/std": 0.0018924169708043337, "sampling/importance_sampling_ratio/max": 1.4372471570968628, "sampling/importance_sampling_ratio/mean": 0.9986727833747864, "sampling/importance_sampling_ratio/min": 0.7778066992759705, "sampling/sampling_logp_difference/max": 0.362729549407959, "sampling/sampling_logp_difference/mean": 0.011521385982632637, "step": 165, "step_time": 1.4701487172860652 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0016393441706895827, "clip_ratio/low_min": 0.0016393441706895827, "clip_ratio/region_mean": 0.0016393441706895827, "completions/clipped_ratio": 0.0, "completions/max_length": 17.0, "completions/max_terminated_length": 17.0, "completions/mean_length": 12.75999927520752, "completions/mean_terminated_length": 12.75999927520752, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.027774377167224883, "epoch": 0.45108695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.02202707901597023, "learning_rate": 5e-05, "loss": -9.761302499100566e-05, "num_tokens": 6139668.0, "reward": 0.37376952171325684, "reward_std": 0.4902029037475586, "rewards/correctness/mean": 0.3799999952316284, "rewards/correctness/std": 0.4903143346309662, "rewards/length_penalty/mean": -0.006230468861758709, "rewards/length_penalty/std": 0.0012382547138258815, "sampling/importance_sampling_ratio/max": 1.684617519378662, "sampling/importance_sampling_ratio/mean": 0.996823251247406, "sampling/importance_sampling_ratio/min": 0.20651641488075256, "sampling/sampling_logp_difference/max": 1.5773754119873047, "sampling/sampling_logp_difference/mean": 0.010566789656877518, "step": 166, "step_time": 1.3450275410432369 }, { "clip_ratio/high_max": 0.00475192591547966, "clip_ratio/high_mean": 0.00475192591547966, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00475192591547966, "completions/clipped_ratio": 0.0, "completions/max_length": 100.0, "completions/max_terminated_length": 100.0, "completions/mean_length": 18.799999237060547, "completions/mean_terminated_length": 18.799999237060547, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 0.17762651443481445, "epoch": 0.453804347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.01814829744398594, "learning_rate": 5e-05, "loss": 0.0020550615154206753, "num_tokens": 6144028.0, "reward": 0.3108203113079071, "reward_std": 0.4736894369125366, "rewards/correctness/mean": 0.3199999928474426, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.00917968712747097, "rewards/length_penalty/std": 0.007274557836353779, "sampling/importance_sampling_ratio/max": 1.3345698118209839, "sampling/importance_sampling_ratio/mean": 0.9937840104103088, "sampling/importance_sampling_ratio/min": 0.5516943335533142, "sampling/sampling_logp_difference/max": 0.5947611331939697, "sampling/sampling_logp_difference/mean": 0.020634151995182037, "step": 167, "step_time": 1.9646003341767937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 45.0, "completions/max_terminated_length": 45.0, "completions/mean_length": 17.0, "completions/mean_terminated_length": 17.0, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.04881806373596191, "epoch": 0.45652173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.015823038294911385, "learning_rate": 5e-05, "loss": 0.0001700791617622599, "num_tokens": 6147718.0, "reward": 0.6316992044448853, "reward_std": 0.4890633523464203, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732054233551, "rewards/length_penalty/mean": -0.00830078125, "rewards/length_penalty/std": 0.005927898455411196, "sampling/importance_sampling_ratio/max": 1.5159095525741577, "sampling/importance_sampling_ratio/mean": 0.9968611598014832, "sampling/importance_sampling_ratio/min": 0.30032309889793396, "sampling/sampling_logp_difference/max": 1.2028963565826416, "sampling/sampling_logp_difference/mean": 0.008511044085025787, "step": 168, "step_time": 1.5832558369729668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.003436876367777586, "clip_ratio/low_min": 0.003436876367777586, "clip_ratio/region_mean": 0.003436876367777586, "completions/clipped_ratio": 0.0, "completions/max_length": 32.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 14.960000038146973, "completions/mean_terminated_length": 14.960000038146973, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.14869228154420852, "epoch": 0.4592391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.007959671318531036, "learning_rate": 5e-05, "loss": 6.60098812659271e-05, "num_tokens": 6151536.0, "reward": 0.032695312052965164, "reward_std": 0.19849225878715515, "rewards/correctness/mean": 0.03999999910593033, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.0073046875186264515, "rewards/length_penalty/std": 0.004218801856040955, "sampling/importance_sampling_ratio/max": 1.6790237426757812, "sampling/importance_sampling_ratio/mean": 0.9950211048126221, "sampling/importance_sampling_ratio/min": 0.43290281295776367, "sampling/sampling_logp_difference/max": 0.8372421264648438, "sampling/sampling_logp_difference/mean": 0.018245313316583633, "step": 169, "step_time": 1.4463021627161652 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 23.0, "completions/max_terminated_length": 23.0, "completions/mean_length": 13.559999465942383, "completions/mean_terminated_length": 13.559999465942383, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.09101279675960541, "epoch": 0.46195652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.007013222668319941, "learning_rate": 5e-05, "loss": 0.00019808998331427574, "num_tokens": 6154464.0, "reward": 0.19337889552116394, "reward_std": 0.40469789505004883, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.006621093954890966, "rewards/length_penalty/std": 0.0022061828058212996, "sampling/importance_sampling_ratio/max": 1.8789424896240234, "sampling/importance_sampling_ratio/mean": 0.9956738948822021, "sampling/importance_sampling_ratio/min": 0.6864051222801208, "sampling/sampling_logp_difference/max": 0.6307090520858765, "sampling/sampling_logp_difference/mean": 0.010942891240119934, "step": 170, "step_time": 1.4700431821402162 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13.0, "completions/max_terminated_length": 13.0, "completions/mean_length": 10.59999942779541, "completions/mean_terminated_length": 10.59999942779541, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.10382853001356125, "epoch": 0.46467391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.0058113946579396725, "learning_rate": 5e-05, "loss": -8.194765541702509e-06, "num_tokens": 6157764.0, "reward": 0.374824196100235, "reward_std": 0.4905463755130768, "rewards/correctness/mean": 0.3799999952316284, "rewards/correctness/std": 0.4903143346309662, "rewards/length_penalty/mean": -0.005175781436264515, "rewards/length_penalty/std": 0.0005918862880207598, "sampling/importance_sampling_ratio/max": 1.7519868612289429, "sampling/importance_sampling_ratio/mean": 1.000873327255249, "sampling/importance_sampling_ratio/min": 0.7990619540214539, "sampling/sampling_logp_difference/max": 0.5607504844665527, "sampling/sampling_logp_difference/mean": 0.009887173771858215, "step": 171, "step_time": 1.3336135821882635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16.0, "completions/max_terminated_length": 16.0, "completions/mean_length": 13.65999984741211, "completions/mean_terminated_length": 13.65999984741211, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 0.022951000183820725, "epoch": 0.4673913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.0066406941041350365, "learning_rate": 5e-05, "loss": 9.038255666382611e-05, "num_tokens": 6161947.0, "reward": 0.5933300852775574, "reward_std": 0.4950695335865021, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.006669921800494194, "rewards/length_penalty/std": 0.0007487484253942966, "sampling/importance_sampling_ratio/max": 1.078048825263977, "sampling/importance_sampling_ratio/mean": 0.9990386366844177, "sampling/importance_sampling_ratio/min": 0.817091703414917, "sampling/sampling_logp_difference/max": 0.20200395584106445, "sampling/sampling_logp_difference/mean": 0.0020261311437934637, "step": 172, "step_time": 1.4464820218272507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12.0, "completions/max_terminated_length": 12.0, "completions/mean_length": 10.819999694824219, "completions/mean_terminated_length": 10.819999694824219, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.09757410287857056, "epoch": 0.4701086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.012097127735614777, "learning_rate": 5e-05, "loss": 6.267993012443185e-05, "num_tokens": 6165778.0, "reward": 0.3947167992591858, "reward_std": 0.4950006306171417, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.005283202975988388, "rewards/length_penalty/std": 0.00036499652196653187, "sampling/importance_sampling_ratio/max": 1.1195776462554932, "sampling/importance_sampling_ratio/mean": 0.9954991936683655, "sampling/importance_sampling_ratio/min": 0.7336348295211792, "sampling/sampling_logp_difference/max": 0.30974388122558594, "sampling/sampling_logp_difference/mean": 0.0087917884811759, "step": 173, "step_time": 1.8255305797792971 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0015267175622284412, "clip_ratio/low_min": 0.0015267175622284412, "clip_ratio/region_mean": 0.0015267175622284412, "completions/clipped_ratio": 0.0, "completions/max_length": 19.0, "completions/max_terminated_length": 19.0, "completions/mean_length": 12.679999351501465, "completions/mean_terminated_length": 12.679999351501465, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.03661280572414398, "epoch": 0.47282608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.014036361128091812, "learning_rate": 5e-05, "loss": 8.49634816404432e-05, "num_tokens": 6168652.0, "reward": 0.3938085734844208, "reward_std": 0.4957515597343445, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.006191406399011612, "rewards/length_penalty/std": 0.0016431285766884685, "sampling/importance_sampling_ratio/max": 1.236996054649353, "sampling/importance_sampling_ratio/mean": 0.9974590539932251, "sampling/importance_sampling_ratio/min": 0.8568878173828125, "sampling/sampling_logp_difference/max": 0.21268582344055176, "sampling/sampling_logp_difference/mean": 0.003560564247891307, "step": 174, "step_time": 1.3956028218381107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 18.0, "completions/max_terminated_length": 18.0, "completions/mean_length": 12.119999885559082, "completions/mean_terminated_length": 12.119999885559082, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.09595631361007691, "epoch": 0.47554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.008084799163043499, "learning_rate": 5e-05, "loss": 0.0002971684734802693, "num_tokens": 6172338.0, "reward": 0.19408202171325684, "reward_std": 0.404585599899292, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.0059179686941206455, "rewards/length_penalty/std": 0.0013036435702815652, "sampling/importance_sampling_ratio/max": 1.2564702033996582, "sampling/importance_sampling_ratio/mean": 0.9970070123672485, "sampling/importance_sampling_ratio/min": 0.6558789014816284, "sampling/sampling_logp_difference/max": 0.42177915573120117, "sampling/sampling_logp_difference/mean": 0.009849517606198788, "step": 175, "step_time": 1.3767987717874348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16.0, "completions/max_terminated_length": 16.0, "completions/mean_length": 12.559999465942383, "completions/mean_terminated_length": 12.559999465942383, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.04389088936150074, "epoch": 0.4782608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.006167205050587654, "learning_rate": 5e-05, "loss": 8.614466059952974e-05, "num_tokens": 6175606.0, "reward": 0.19386717677116394, "reward_std": 0.4042009115219116, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.006132812704890966, "rewards/length_penalty/std": 0.0012088270159438252, "sampling/importance_sampling_ratio/max": 1.111163854598999, "sampling/importance_sampling_ratio/mean": 0.999321460723877, "sampling/importance_sampling_ratio/min": 0.899779200553894, "sampling/sampling_logp_difference/max": 0.1056058406829834, "sampling/sampling_logp_difference/mean": 0.0016725992318242788, "step": 176, "step_time": 1.3660327643156052 }, { "clip_ratio/high_max": 0.0018348623067140578, "clip_ratio/high_mean": 0.0018348623067140578, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018348623067140578, "completions/clipped_ratio": 0.0, "completions/max_length": 13.0, "completions/max_terminated_length": 13.0, "completions/mean_length": 10.84000015258789, "completions/mean_terminated_length": 10.84000015258789, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.11361469924449921, "epoch": 0.48097826086956524, "frac_reward_zero_std": 0.0, "grad_norm": 0.014467968605458736, "learning_rate": 5e-05, "loss": -1.2248288840055466e-05, "num_tokens": 6178378.0, "reward": 0.2947070300579071, "reward_std": 0.46285873651504517, "rewards/correctness/mean": 0.30000001192092896, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.005292968824505806, "rewards/length_penalty/std": 0.0005697705782949924, "sampling/importance_sampling_ratio/max": 1.2576252222061157, "sampling/importance_sampling_ratio/mean": 0.9976921677589417, "sampling/importance_sampling_ratio/min": 0.6826483607292175, "sampling/sampling_logp_difference/max": 0.3817753791809082, "sampling/sampling_logp_difference/mean": 0.01007007621228695, "step": 177, "step_time": 1.3444288549944758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00500610526651144, "clip_ratio/low_min": 0.00500610526651144, "clip_ratio/region_mean": 0.00500610526651144, "completions/clipped_ratio": 0.0, "completions/max_length": 18.0, "completions/max_terminated_length": 18.0, "completions/mean_length": 11.75999927520752, "completions/mean_terminated_length": 11.75999927520752, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.03965783976018429, "epoch": 0.483695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.018577076494693756, "learning_rate": 5e-05, "loss": 0.00013596841017715633, "num_tokens": 6181426.0, "reward": 0.5942577719688416, "reward_std": 0.49553337693214417, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.005742187611758709, "rewards/length_penalty/std": 0.0012807496823370457, "sampling/importance_sampling_ratio/max": 1.6979511976242065, "sampling/importance_sampling_ratio/mean": 1.000203251838684, "sampling/importance_sampling_ratio/min": 0.6369588971138, "sampling/sampling_logp_difference/max": 0.5294222831726074, "sampling/sampling_logp_difference/mean": 0.00880514457821846, "step": 178, "step_time": 1.3555466106627136 }, { "clip_ratio/high_max": 0.003141167759895325, "clip_ratio/high_mean": 0.003141167759895325, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003141167759895325, "completions/clipped_ratio": 0.0, "completions/max_length": 19.0, "completions/max_terminated_length": 19.0, "completions/mean_length": 12.460000038146973, "completions/mean_terminated_length": 12.460000038146973, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.08259178996086121, "epoch": 0.48641304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.019597185775637627, "learning_rate": 5e-05, "loss": 0.00048097348189912736, "num_tokens": 6184849.0, "reward": -0.0060839843936264515, "reward_std": 0.001353344414383173, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0060839843936264515, "rewards/length_penalty/std": 0.001353344414383173, "sampling/importance_sampling_ratio/max": 1.4630529880523682, "sampling/importance_sampling_ratio/mean": 0.9944373965263367, "sampling/importance_sampling_ratio/min": 0.7110886573791504, "sampling/sampling_logp_difference/max": 0.3805253505706787, "sampling/sampling_logp_difference/mean": 0.009516164660453796, "step": 179, "step_time": 1.3651126499753445 }, { "clip_ratio/high_max": 0.004180645197629928, "clip_ratio/high_mean": 0.004180645197629928, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004180645197629928, "completions/clipped_ratio": 0.0, "completions/max_length": 24.0, "completions/max_terminated_length": 24.0, "completions/mean_length": 13.319999694824219, "completions/mean_terminated_length": 13.319999694824219, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.1470176860690117, "epoch": 0.4891304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.01228439249098301, "learning_rate": 5e-05, "loss": 9.731328464113176e-05, "num_tokens": 6189975.0, "reward": 0.1134960874915123, "reward_std": 0.3288118839263916, "rewards/correctness/mean": 0.11999999731779099, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.006503906100988388, "rewards/length_penalty/std": 0.002237455453723669, "sampling/importance_sampling_ratio/max": 1.2735097408294678, "sampling/importance_sampling_ratio/mean": 0.9983915686607361, "sampling/importance_sampling_ratio/min": 0.6723344922065735, "sampling/sampling_logp_difference/max": 0.3969993591308594, "sampling/sampling_logp_difference/mean": 0.011573792435228825, "step": 180, "step_time": 1.5991302751936018 }, { "clip_ratio/high_max": 0.0018018018454313278, "clip_ratio/high_mean": 0.0018018018454313278, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018018018454313278, "completions/clipped_ratio": 0.0, "completions/max_length": 20.0, "completions/max_terminated_length": 20.0, "completions/mean_length": 11.300000190734863, "completions/mean_terminated_length": 11.300000190734863, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.0894030824303627, "epoch": 0.49184782608695654, "frac_reward_zero_std": 0.0, "grad_norm": 0.009084840305149555, "learning_rate": 5e-05, "loss": 0.0003331962216179818, "num_tokens": 6193700.0, "reward": 0.23448242247104645, "reward_std": 0.43150392174720764, "rewards/correctness/mean": 0.23999999463558197, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.0055175782181322575, "rewards/length_penalty/std": 0.0011970564955845475, "sampling/importance_sampling_ratio/max": 1.4459896087646484, "sampling/importance_sampling_ratio/mean": 0.9954925775527954, "sampling/importance_sampling_ratio/min": 0.17653107643127441, "sampling/sampling_logp_difference/max": 1.7342582941055298, "sampling/sampling_logp_difference/mean": 0.01727665774524212, "step": 181, "step_time": 1.4905728702433407 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.001666666753590107, "clip_ratio/low_min": 0.001666666753590107, "clip_ratio/region_mean": 0.001666666753590107, "completions/clipped_ratio": 0.0, "completions/max_length": 14.0, "completions/max_terminated_length": 14.0, "completions/mean_length": 11.699999809265137, "completions/mean_terminated_length": 11.699999809265137, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.11428584754467011, "epoch": 0.4945652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.010212992317974567, "learning_rate": 5e-05, "loss": 0.00020022568060085177, "num_tokens": 6197125.0, "reward": 0.1742871105670929, "reward_std": 0.38825035095214844, "rewards/correctness/mean": 0.18000000715255737, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.0057128905318677425, "rewards/length_penalty/std": 0.000726585800293833, "sampling/importance_sampling_ratio/max": 1.5589516162872314, "sampling/importance_sampling_ratio/mean": 0.9978711605072021, "sampling/importance_sampling_ratio/min": 0.5401617288589478, "sampling/sampling_logp_difference/max": 0.6158866882324219, "sampling/sampling_logp_difference/mean": 0.010786871425807476, "step": 182, "step_time": 1.514930889941752 }, { "clip_ratio/high_max": 0.0016260161995887757, "clip_ratio/high_mean": 0.0016260161995887757, "clip_ratio/low_mean": 0.006719549000263214, "clip_ratio/low_min": 0.006719549000263214, "clip_ratio/region_mean": 0.00834556519985199, "completions/clipped_ratio": 0.0, "completions/max_length": 17.0, "completions/max_terminated_length": 17.0, "completions/mean_length": 11.880000114440918, "completions/mean_terminated_length": 11.880000114440918, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.06704182587563992, "epoch": 0.49728260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.010296587832272053, "learning_rate": 5e-05, "loss": 0.0001108224387280643, "num_tokens": 6200009.0, "reward": 0.5341991782188416, "reward_std": 0.5032960176467896, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.0058007813058793545, "rewards/length_penalty/std": 0.0012923979666084051, "sampling/importance_sampling_ratio/max": 2.955549716949463, "sampling/importance_sampling_ratio/mean": 1.0004843473434448, "sampling/importance_sampling_ratio/min": 0.46944674849510193, "sampling/sampling_logp_difference/max": 1.0836846828460693, "sampling/sampling_logp_difference/mean": 0.0124569833278656, "step": 183, "step_time": 1.4682957818731666 }, { "clip_ratio/high_max": 0.0018348623067140578, "clip_ratio/high_mean": 0.0018348623067140578, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018348623067140578, "completions/clipped_ratio": 0.0, "completions/max_length": 13.0, "completions/max_terminated_length": 13.0, "completions/mean_length": 10.979999542236328, "completions/mean_terminated_length": 10.979999542236328, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.07332217693328857, "epoch": 0.5, "frac_reward_zero_std": 0.0, "grad_norm": 0.014040185138583183, "learning_rate": 5e-05, "loss": 0.00010328828648198396, "num_tokens": 6203308.0, "reward": -0.005361327901482582, "reward_std": 0.0005447104922495782, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.005361327901482582, "rewards/length_penalty/std": 0.0005447104922495782, "sampling/importance_sampling_ratio/max": 1.4190045595169067, "sampling/importance_sampling_ratio/mean": 1.000106692314148, "sampling/importance_sampling_ratio/min": 0.2903344929218292, "sampling/sampling_logp_difference/max": 1.2367215156555176, "sampling/sampling_logp_difference/mean": 0.014944715425372124, "step": 184, "step_time": 1.4993287862744182 }, { "clip_ratio/high_max": 0.0010416666977107526, "clip_ratio/high_mean": 0.0010416666977107526, "clip_ratio/low_mean": 0.0010416666977107526, "clip_ratio/low_min": 0.0010416666977107526, "clip_ratio/region_mean": 0.002083333395421505, "completions/clipped_ratio": 0.0, "completions/max_length": 42.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 17.239999771118164, "completions/mean_terminated_length": 17.239999771118164, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 0.12270538061857224, "epoch": 0.5027173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.010194046422839165, "learning_rate": 5e-05, "loss": 0.0010757609270513058, "num_tokens": 6207890.0, "reward": -0.008417968638241291, "reward_std": 0.005005396902561188, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.008417968638241291, "rewards/length_penalty/std": 0.005005396902561188, "sampling/importance_sampling_ratio/max": 2.1810836791992188, "sampling/importance_sampling_ratio/mean": 0.9978315234184265, "sampling/importance_sampling_ratio/min": 0.6044803857803345, "sampling/sampling_logp_difference/max": 0.7798217535018921, "sampling/sampling_logp_difference/mean": 0.018191643059253693, "step": 185, "step_time": 2.064146334771067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16.0, "completions/max_terminated_length": 16.0, "completions/mean_length": 11.960000038146973, "completions/mean_terminated_length": 11.960000038146973, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.04249002113938331, "epoch": 0.5054347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.010809618048369884, "learning_rate": 5e-05, "loss": 2.4108157958835363e-06, "num_tokens": 6212558.0, "reward": 0.1741601526737213, "reward_std": 0.3883109986782074, "rewards/correctness/mean": 0.18000000715255737, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.0058398437686264515, "rewards/length_penalty/std": 0.0010009704856202006, "sampling/importance_sampling_ratio/max": 2.3153789043426514, "sampling/importance_sampling_ratio/mean": 0.9990188479423523, "sampling/importance_sampling_ratio/min": 0.5857009887695312, "sampling/sampling_logp_difference/max": 0.8395733833312988, "sampling/sampling_logp_difference/mean": 0.0076761506497859955, "step": 186, "step_time": 1.5036294928286225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0036375662311911583, "clip_ratio/low_min": 0.0036375662311911583, "clip_ratio/region_mean": 0.0036375662311911583, "completions/clipped_ratio": 0.0, "completions/max_length": 13.0, "completions/max_terminated_length": 13.0, "completions/mean_length": 11.25999927520752, "completions/mean_terminated_length": 11.25999927520752, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.06017339825630188, "epoch": 0.5081521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.02455165982246399, "learning_rate": 5e-05, "loss": 3.779780672630295e-05, "num_tokens": 6216181.0, "reward": 0.19450195133686066, "reward_std": 0.40363213419914246, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.005498046986758709, "rewards/length_penalty/std": 0.0006065831403248012, "sampling/importance_sampling_ratio/max": 1.5202230215072632, "sampling/importance_sampling_ratio/mean": 1.0005755424499512, "sampling/importance_sampling_ratio/min": 0.6220064759254456, "sampling/sampling_logp_difference/max": 0.47480475902557373, "sampling/sampling_logp_difference/mean": 0.013067500665783882, "step": 187, "step_time": 1.3525943402200937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16.0, "completions/max_terminated_length": 16.0, "completions/mean_length": 11.679999351501465, "completions/mean_terminated_length": 11.679999351501465, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.054110404849052426, "epoch": 0.5108695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.011299094185233116, "learning_rate": 5e-05, "loss": -0.00019597375649027526, "num_tokens": 6220685.0, "reward": 0.3942968547344208, "reward_std": 0.49424001574516296, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.005703125149011612, "rewards/length_penalty/std": 0.0010177461663261056, "sampling/importance_sampling_ratio/max": 1.1664270162582397, "sampling/importance_sampling_ratio/mean": 0.9997606873512268, "sampling/importance_sampling_ratio/min": 0.6920753121376038, "sampling/sampling_logp_difference/max": 0.3680605888366699, "sampling/sampling_logp_difference/mean": 0.004667541012167931, "step": 188, "step_time": 1.4143610841128975 }, { "clip_ratio/high_max": 0.0016393441706895827, "clip_ratio/high_mean": 0.0016393441706895827, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016393441706895827, "completions/clipped_ratio": 0.0, "completions/max_length": 18.0, "completions/max_terminated_length": 18.0, "completions/mean_length": 12.09999942779541, "completions/mean_terminated_length": 12.09999942779541, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.11203449964523315, "epoch": 0.5135869565217391, "frac_reward_zero_std": 0.0, "grad_norm": 0.011153757572174072, "learning_rate": 5e-05, "loss": 2.8730537451338023e-05, "num_tokens": 6224390.0, "reward": 0.2540917992591858, "reward_std": 0.44296059012413025, "rewards/correctness/mean": 0.25999999046325684, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.005908203311264515, "rewards/length_penalty/std": 0.0010772491805255413, "sampling/importance_sampling_ratio/max": 1.6477757692337036, "sampling/importance_sampling_ratio/mean": 0.9972930550575256, "sampling/importance_sampling_ratio/min": 0.7198603749275208, "sampling/sampling_logp_difference/max": 0.49942636489868164, "sampling/sampling_logp_difference/mean": 0.01260648388415575, "step": 189, "step_time": 1.397800819016993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0018518518656492234, "clip_ratio/low_min": 0.0018518518656492234, "clip_ratio/region_mean": 0.0018518518656492234, "completions/clipped_ratio": 0.0, "completions/max_length": 12.0, "completions/max_terminated_length": 12.0, "completions/mean_length": 10.760000228881836, "completions/mean_terminated_length": 10.760000228881836, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.06086069643497467, "epoch": 0.5163043478260869, "frac_reward_zero_std": 0.0, "grad_norm": 0.02036065049469471, "learning_rate": 5e-05, "loss": 8.105181768769398e-05, "num_tokens": 6227488.0, "reward": 0.19474609196186066, "reward_std": 0.4037552773952484, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.005253906361758709, "rewards/length_penalty/std": 0.000376415642676875, "sampling/importance_sampling_ratio/max": 1.4321415424346924, "sampling/importance_sampling_ratio/mean": 0.9983711242675781, "sampling/importance_sampling_ratio/min": 0.6117324233055115, "sampling/sampling_logp_difference/max": 0.49146032333374023, "sampling/sampling_logp_difference/mean": 0.008058278821408749, "step": 190, "step_time": 1.3019498747307807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15.0, "completions/max_terminated_length": 15.0, "completions/mean_length": 11.219999313354492, "completions/mean_terminated_length": 11.219999313354492, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.026026188768446444, "epoch": 0.5190217391304348, "frac_reward_zero_std": 0.0, "grad_norm": 0.01458522118628025, "learning_rate": 5e-05, "loss": 5.049306491855532e-05, "num_tokens": 6230359.0, "reward": 0.1945214867591858, "reward_std": 0.40436291694641113, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.0054785157553851604, "rewards/length_penalty/std": 0.0009578992030583322, "sampling/importance_sampling_ratio/max": 1.0809760093688965, "sampling/importance_sampling_ratio/mean": 0.9979611039161682, "sampling/importance_sampling_ratio/min": 0.8442093729972839, "sampling/sampling_logp_difference/max": 0.16935479640960693, "sampling/sampling_logp_difference/mean": 0.0029735560528934, "step": 191, "step_time": 1.3701859309803694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13.0, "completions/max_terminated_length": 13.0, "completions/mean_length": 10.84000015258789, "completions/mean_terminated_length": 10.84000015258789, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.120220248401165, "epoch": 0.5217391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.013222390785813332, "learning_rate": 5e-05, "loss": 0.00018757855286821723, "num_tokens": 6235701.0, "reward": -0.005292968824505806, "reward_std": 0.000515994499437511, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.005292968824505806, "rewards/length_penalty/std": 0.000515994499437511, "sampling/importance_sampling_ratio/max": 1.2311463356018066, "sampling/importance_sampling_ratio/mean": 0.99524986743927, "sampling/importance_sampling_ratio/min": 0.5662293434143066, "sampling/sampling_logp_difference/max": 0.568756103515625, "sampling/sampling_logp_difference/mean": 0.013108277693390846, "step": 192, "step_time": 1.4611065411008894 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13.0, "completions/max_terminated_length": 13.0, "completions/mean_length": 11.0, "completions/mean_terminated_length": 11.0, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.023558325506746768, "epoch": 0.5244565217391305, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 6238381.0, "reward": 0.19462890923023224, "reward_std": 0.4043080508708954, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.00537109375, "rewards/length_penalty/std": 0.0006239028880372643, "sampling/importance_sampling_ratio/max": 1.0198575258255005, "sampling/importance_sampling_ratio/mean": 0.9979141354560852, "sampling/importance_sampling_ratio/min": 0.798582911491394, "sampling/sampling_logp_difference/max": 0.2249164581298828, "sampling/sampling_logp_difference/mean": 0.0022540087811648846, "step": 193, "step_time": 1.384955577319488 }, { "clip_ratio/high_max": 0.0036873405799269674, "clip_ratio/high_mean": 0.0036873405799269674, "clip_ratio/low_mean": 0.00181818176060915, "clip_ratio/low_min": 0.00181818176060915, "clip_ratio/region_mean": 0.005505522340536117, "completions/clipped_ratio": 0.0, "completions/max_length": 12.0, "completions/max_terminated_length": 12.0, "completions/mean_length": 10.699999809265137, "completions/mean_terminated_length": 10.699999809265137, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.05290040671825409, "epoch": 0.5271739130434783, "frac_reward_zero_std": 0.0, "grad_norm": 0.030923819169402122, "learning_rate": 5e-05, "loss": 0.000215167659916915, "num_tokens": 6242576.0, "reward": 0.3347753882408142, "reward_std": 0.47820374369621277, "rewards/correctness/mean": 0.3400000035762787, "rewards/correctness/std": 0.47851812839508057, "rewards/length_penalty/mean": -0.0052246092818677425, "rewards/length_penalty/std": 0.0003852316294796765, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 1.001953363418579, "sampling/importance_sampling_ratio/min": 0.14254866540431976, "sampling/sampling_logp_difference/max": 1.948071837425232, "sampling/sampling_logp_difference/mean": 0.049299318343400955, "step": 194, "step_time": 1.3902816667687148 }, { "clip_ratio/high_max": 0.004845302924513817, "clip_ratio/high_mean": 0.004845302924513817, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004845302924513817, "completions/clipped_ratio": 0.0, "completions/max_length": 15.0, "completions/max_terminated_length": 15.0, "completions/mean_length": 12.59999942779541, "completions/mean_terminated_length": 12.59999942779541, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.08983674570918083, "epoch": 0.529891304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.04137495532631874, "learning_rate": 5e-05, "loss": 5.23802445968613e-05, "num_tokens": 6245566.0, "reward": 0.1338476538658142, "reward_std": 0.35063034296035767, "rewards/correctness/mean": 0.14000000059604645, "rewards/correctness/std": 0.3505098223686218, "rewards/length_penalty/mean": -0.006152343936264515, "rewards/length_penalty/std": 0.0008878294029273093, "sampling/importance_sampling_ratio/max": 2.7264294624328613, "sampling/importance_sampling_ratio/mean": 0.9958686828613281, "sampling/importance_sampling_ratio/min": 0.3633946180343628, "sampling/sampling_logp_difference/max": 1.012265920639038, "sampling/sampling_logp_difference/mean": 0.030177565291523933, "step": 195, "step_time": 1.3612184193916619 }, { "clip_ratio/high_max": 0.0015151515603065492, "clip_ratio/high_mean": 0.0015151515603065492, "clip_ratio/low_mean": 0.001666666753590107, "clip_ratio/low_min": 0.001666666753590107, "clip_ratio/region_mean": 0.003181818313896656, "completions/clipped_ratio": 0.0, "completions/max_length": 30.0, "completions/max_terminated_length": 30.0, "completions/mean_length": 11.75999927520752, "completions/mean_terminated_length": 11.75999927520752, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.08821693509817123, "epoch": 0.532608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.011738491244614124, "learning_rate": 5e-05, "loss": 0.0008821445517241955, "num_tokens": 6248624.0, "reward": -0.005742187146097422, "reward_std": 0.001468360424041748, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.005742187611758709, "rewards/length_penalty/std": 0.001468360424041748, "sampling/importance_sampling_ratio/max": 1.387971043586731, "sampling/importance_sampling_ratio/mean": 0.9953455328941345, "sampling/importance_sampling_ratio/min": 0.36234790086746216, "sampling/sampling_logp_difference/max": 1.015150547027588, "sampling/sampling_logp_difference/mean": 0.013325001113116741, "step": 196, "step_time": 1.410275483969599 }, { "clip_ratio/high_max": 0.0018348623067140578, "clip_ratio/high_mean": 0.0018348623067140578, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018348623067140578, "completions/clipped_ratio": 0.0, "completions/max_length": 29.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 11.139999389648438, "completions/mean_terminated_length": 11.139999389648438, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.11148715764284134, "epoch": 0.5353260869565217, "frac_reward_zero_std": 0.0, "grad_norm": 0.009937284514307976, "learning_rate": 5e-05, "loss": 0.0011085595469921827, "num_tokens": 6252581.0, "reward": 0.19456054270267487, "reward_std": 0.404344379901886, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.005439453292638063, "rewards/length_penalty/std": 0.001370493322610855, "sampling/importance_sampling_ratio/max": 1.6792072057724, "sampling/importance_sampling_ratio/mean": 0.9960465431213379, "sampling/importance_sampling_ratio/min": 0.23884545266628265, "sampling/sampling_logp_difference/max": 1.431938648223877, "sampling/sampling_logp_difference/mean": 0.02536919340491295, "step": 197, "step_time": 1.4341971618123353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14.0, "completions/max_terminated_length": 14.0, "completions/mean_length": 10.739999771118164, "completions/mean_terminated_length": 10.739999771118164, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.029788795113563537, "epoch": 0.5380434782608695, "frac_reward_zero_std": 0.0, "grad_norm": 0.006434222217649221, "learning_rate": 5e-05, "loss": 0.00022055389126762748, "num_tokens": 6259028.0, "reward": 0.39475584030151367, "reward_std": 0.49476706981658936, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.005244140513241291, "rewards/length_penalty/std": 0.00032433951855637133, "sampling/importance_sampling_ratio/max": 1.139495611190796, "sampling/importance_sampling_ratio/mean": 0.9981493949890137, "sampling/importance_sampling_ratio/min": 0.6036344170570374, "sampling/sampling_logp_difference/max": 0.504786491394043, "sampling/sampling_logp_difference/mean": 0.006575726438313723, "step": 198, "step_time": 2.050717952195555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16.0, "completions/max_terminated_length": 16.0, "completions/mean_length": 12.0, "completions/mean_terminated_length": 12.0, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.014517558831721544, "epoch": 0.5407608695652174, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 6262318.0, "reward": 0.3941406309604645, "reward_std": 0.49406662583351135, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.005859375, "rewards/length_penalty/std": 0.0010346259223297238, "sampling/importance_sampling_ratio/max": 1.1886876821517944, "sampling/importance_sampling_ratio/mean": 0.9984161853790283, "sampling/importance_sampling_ratio/min": 0.8961021900177002, "sampling/sampling_logp_difference/max": 0.1728498935699463, "sampling/sampling_logp_difference/mean": 0.002286211820319295, "step": 199, "step_time": 1.3379682791419327 }, { "clip_ratio/high_max": 0.001587301678955555, "clip_ratio/high_mean": 0.001587301678955555, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001587301678955555, "completions/clipped_ratio": 0.0, "completions/max_length": 19.0, "completions/max_terminated_length": 19.0, "completions/mean_length": 12.079999923706055, "completions/mean_terminated_length": 12.079999923706055, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.11163323372602463, "epoch": 0.5434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.00912521407008171, "learning_rate": 5e-05, "loss": 0.00011058378731831908, "num_tokens": 6267182.0, "reward": -0.005898437462747097, "reward_std": 0.0016084114322438836, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.005898437462747097, "rewards/length_penalty/std": 0.0016084115486592054, "sampling/importance_sampling_ratio/max": 1.5538405179977417, "sampling/importance_sampling_ratio/mean": 0.9894649386405945, "sampling/importance_sampling_ratio/min": 0.3413477838039398, "sampling/sampling_logp_difference/max": 1.0748534202575684, "sampling/sampling_logp_difference/mean": 0.01791464537382126, "step": 200, "step_time": 1.502127684885636 } ], "logging_steps": 1, "max_steps": 200, "num_input_tokens_seen": 6267182, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 10, "trial_name": null, "trial_params": null }