{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.2717391304347826, "eval_steps": 500, "global_step": 100, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1671.2799072265625, "completions/mean_terminated_length": 1398.4827880859375, "completions/min_length": 992.0, "completions/min_terminated_length": 992.0, "entropy": 0.21248915791511536, "epoch": 0.002717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.01622791402041912, "learning_rate": 0.0, "loss": 0.06695985794067383, "num_tokens": 86304.0, "reward": -0.21605467796325684, "reward_std": 0.656523585319519, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.8160547018051147, "rewards/length_penalty/std": 0.18964654207229614, "sampling/importance_sampling_ratio/max": 1.5222556591033936, "sampling/importance_sampling_ratio/mean": 0.9927070140838623, "sampling/importance_sampling_ratio/min": 0.6435883045196533, "sampling/sampling_logp_difference/max": 0.44069600105285645, "sampling/sampling_logp_difference/mean": 0.013988969847559929, "step": 1, "step_time": 27.388843874214217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6800000071525574, "completions/max_length": 2048.0, "completions/max_terminated_length": 1991.0, "completions/mean_length": 1941.47998046875, "completions/mean_terminated_length": 1715.125, "completions/min_length": 1079.0, "completions/min_terminated_length": 1079.0, "entropy": 0.2905632793903351, "epoch": 0.005434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.017553681507706642, "learning_rate": 5e-06, "loss": 0.06798752397298813, "num_tokens": 186618.0, "reward": -0.7279882431030273, "reward_std": 0.4917677640914917, "rewards/correctness/mean": 0.2199999988079071, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.9479882717132568, "rewards/length_penalty/std": 0.10518474876880646, "sampling/importance_sampling_ratio/max": 1.450933814048767, "sampling/importance_sampling_ratio/mean": 0.9901063442230225, "sampling/importance_sampling_ratio/min": 0.3688630163669586, "sampling/sampling_logp_difference/max": 0.9973299503326416, "sampling/sampling_logp_difference/mean": 0.01780563034117222, "step": 2, "step_time": 29.02393381111324 }, { "clip_ratio/high_max": 0.0009701523347757756, "clip_ratio/high_mean": 0.0009701523347757756, "clip_ratio/low_mean": 6.219160495675169e-05, "clip_ratio/low_min": 6.219160495675169e-05, "clip_ratio/region_mean": 0.0010323439724743365, "completions/clipped_ratio": 0.3199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 1901.0, "completions/mean_length": 1612.3199462890625, "completions/mean_terminated_length": 1407.2940673828125, "completions/min_length": 745.0, "completions/min_terminated_length": 745.0, "entropy": 0.255071622133255, "epoch": 0.008152173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.017932165414094925, "learning_rate": 1e-05, "loss": 0.13346610963344574, "num_tokens": 269504.0, "reward": -0.08726562559604645, "reward_std": 0.6200677156448364, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.7872655987739563, "rewards/length_penalty/std": 0.19988179206848145, "sampling/importance_sampling_ratio/max": 1.5381298065185547, "sampling/importance_sampling_ratio/mean": 0.9911423921585083, "sampling/importance_sampling_ratio/min": 0.6395388841629028, "sampling/sampling_logp_difference/max": 0.4470078945159912, "sampling/sampling_logp_difference/mean": 0.016371745616197586, "step": 3, "step_time": 27.040046802023426 }, { "clip_ratio/high_max": 0.0007077743241097779, "clip_ratio/high_mean": 0.0007077743241097779, "clip_ratio/low_mean": 6.867761694593356e-05, "clip_ratio/low_min": 6.867761694593356e-05, "clip_ratio/region_mean": 0.0007764519308693707, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 1882.0, "completions/mean_length": 1489.419921875, "completions/mean_terminated_length": 1250.028564453125, "completions/min_length": 626.0, "completions/min_terminated_length": 626.0, "entropy": 0.24563565850257874, "epoch": 0.010869565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.017823319882154465, "learning_rate": 1.5e-05, "loss": 0.09811276942491531, "num_tokens": 346875.0, "reward": -0.007255859207361937, "reward_std": 0.6479107141494751, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.7272558808326721, "rewards/length_penalty/std": 0.2412886619567871, "sampling/importance_sampling_ratio/max": 1.5703521966934204, "sampling/importance_sampling_ratio/mean": 0.9914191961288452, "sampling/importance_sampling_ratio/min": 0.5496033430099487, "sampling/sampling_logp_difference/max": 0.5985584259033203, "sampling/sampling_logp_difference/mean": 0.016086140647530556, "step": 4, "step_time": 27.06473581609316 }, { "clip_ratio/high_max": 0.0014693949837237597, "clip_ratio/high_mean": 0.0014693949837237597, "clip_ratio/low_mean": 2.9735209682257847e-05, "clip_ratio/low_min": 2.9735209682257847e-05, "clip_ratio/region_mean": 0.0014991301926784218, "completions/clipped_ratio": 0.8199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 1909.0, "completions/mean_length": 1955.3800048828125, "completions/mean_terminated_length": 1533.4444580078125, "completions/min_length": 1097.0, "completions/min_terminated_length": 1097.0, "entropy": 0.2879446566104889, "epoch": 0.01358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.015369652770459652, "learning_rate": 2e-05, "loss": 0.05972106009721756, "num_tokens": 448354.0, "reward": -0.7747753858566284, "reward_std": 0.4882567822933197, "rewards/correctness/mean": 0.18000000715255737, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.9547753930091858, "rewards/length_penalty/std": 0.11000200361013412, "sampling/importance_sampling_ratio/max": 2.131683349609375, "sampling/importance_sampling_ratio/mean": 0.9901448488235474, "sampling/importance_sampling_ratio/min": 0.6602146625518799, "sampling/sampling_logp_difference/max": 0.7569119930267334, "sampling/sampling_logp_difference/mean": 0.01812317781150341, "step": 5, "step_time": 29.137280595256016 }, { "clip_ratio/high_max": 0.000520571404194925, "clip_ratio/high_mean": 0.000520571404194925, "clip_ratio/low_mean": 0.00017908125300891697, "clip_ratio/low_min": 0.00017908125300891697, "clip_ratio/region_mean": 0.0006996526441071183, "completions/clipped_ratio": 0.4599999785423279, "completions/max_length": 2048.0, "completions/max_terminated_length": 1941.0, "completions/mean_length": 1660.0, "completions/mean_terminated_length": 1329.4814453125, "completions/min_length": 723.0, "completions/min_terminated_length": 723.0, "entropy": 0.2539885014295578, "epoch": 0.016304347826086956, "frac_reward_zero_std": 0.0, "grad_norm": 0.01736672781407833, "learning_rate": 2.5e-05, "loss": 0.08885529637336731, "num_tokens": 534344.0, "reward": -0.23054687678813934, "reward_std": 0.6794751286506653, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.810546875, "rewards/length_penalty/std": 0.22522568702697754, "sampling/importance_sampling_ratio/max": 1.742319941520691, "sampling/importance_sampling_ratio/mean": 0.9909853339195251, "sampling/importance_sampling_ratio/min": 0.5412515997886658, "sampling/sampling_logp_difference/max": 0.6138710975646973, "sampling/sampling_logp_difference/mean": 0.016689306125044823, "step": 6, "step_time": 27.337108821840957 }, { "clip_ratio/high_max": 0.0006314931088127196, "clip_ratio/high_mean": 0.0006314931088127196, "clip_ratio/low_mean": 7.634440262336284e-05, "clip_ratio/low_min": 7.634440262336284e-05, "clip_ratio/region_mean": 0.0007078375143464655, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 2029.0, "completions/mean_length": 1579.8399658203125, "completions/mean_terminated_length": 1379.199951171875, "completions/min_length": 803.0, "completions/min_terminated_length": 803.0, "entropy": 0.2385493516921997, "epoch": 0.019021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.016621841117739677, "learning_rate": 3e-05, "loss": 0.08834289014339447, "num_tokens": 615566.0, "reward": -0.05140624940395355, "reward_std": 0.6118109226226807, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.7714062333106995, "rewards/length_penalty/std": 0.19486749172210693, "sampling/importance_sampling_ratio/max": 1.4593793153762817, "sampling/importance_sampling_ratio/mean": 0.9915305972099304, "sampling/importance_sampling_ratio/min": 0.651093065738678, "sampling/sampling_logp_difference/max": 0.42910265922546387, "sampling/sampling_logp_difference/mean": 0.015702536329627037, "step": 7, "step_time": 27.038635928882286 }, { "clip_ratio/high_max": 0.0008715830335859209, "clip_ratio/high_mean": 0.0008715830335859209, "clip_ratio/low_mean": 0.00011999263515463098, "clip_ratio/low_min": 0.00011999263515463098, "clip_ratio/region_mean": 0.0009915756527334452, "completions/clipped_ratio": 0.5999999642372131, "completions/max_length": 2048.0, "completions/max_terminated_length": 1938.0, "completions/mean_length": 1801.239990234375, "completions/mean_terminated_length": 1431.0999755859375, "completions/min_length": 930.0, "completions/min_terminated_length": 930.0, "entropy": 0.21174302995204924, "epoch": 0.021739130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.01676737703382969, "learning_rate": 3.5e-05, "loss": 0.08332858979701996, "num_tokens": 707778.0, "reward": -0.47951170802116394, "reward_std": 0.6519015431404114, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.8795117139816284, "rewards/length_penalty/std": 0.18039129674434662, "sampling/importance_sampling_ratio/max": 1.4509235620498657, "sampling/importance_sampling_ratio/mean": 0.9925462007522583, "sampling/importance_sampling_ratio/min": 0.052912116050720215, "sampling/sampling_logp_difference/max": 2.9391229152679443, "sampling/sampling_logp_difference/mean": 0.014070631936192513, "step": 8, "step_time": 28.223328100983053 }, { "clip_ratio/high_max": 0.0006956349650863558, "clip_ratio/high_mean": 0.0006956349650863558, "clip_ratio/low_mean": 8.642674874863588e-05, "clip_ratio/low_min": 8.642674874863588e-05, "clip_ratio/region_mean": 0.0007820617058314383, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1994.0, "completions/mean_length": 1600.239990234375, "completions/mean_terminated_length": 1442.9189453125, "completions/min_length": 785.0, "completions/min_terminated_length": 785.0, "entropy": 0.23202865421772004, "epoch": 0.024456521739130436, "frac_reward_zero_std": 0.0, "grad_norm": 0.017678245902061462, "learning_rate": 4e-05, "loss": 0.09870695322751999, "num_tokens": 790260.0, "reward": -0.20136718451976776, "reward_std": 0.6241466403007507, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.7813671827316284, "rewards/length_penalty/std": 0.1977044939994812, "sampling/importance_sampling_ratio/max": 1.5457746982574463, "sampling/importance_sampling_ratio/mean": 0.991839587688446, "sampling/importance_sampling_ratio/min": 0.556106448173523, "sampling/sampling_logp_difference/max": 0.5867955684661865, "sampling/sampling_logp_difference/mean": 0.01518256589770317, "step": 9, "step_time": 27.120734592201188 }, { "clip_ratio/high_max": 0.0007978752721101046, "clip_ratio/high_mean": 0.0007978752721101046, "clip_ratio/low_mean": 7.057307666400448e-05, "clip_ratio/low_min": 7.057307666400448e-05, "clip_ratio/region_mean": 0.0008684483473189175, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 2048.0, "completions/mean_length": 1484.0999755859375, "completions/mean_terminated_length": 1264.8055419921875, "completions/min_length": 668.0, "completions/min_terminated_length": 668.0, "entropy": 0.24629398584365844, "epoch": 0.02717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.017371023073792458, "learning_rate": 4.5e-05, "loss": 0.09812771528959274, "num_tokens": 869285.0, "reward": -0.024658203125, "reward_std": 0.647922933101654, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.724658191204071, "rewards/length_penalty/std": 0.2301420420408249, "sampling/importance_sampling_ratio/max": 1.4670653343200684, "sampling/importance_sampling_ratio/mean": 0.9915652275085449, "sampling/importance_sampling_ratio/min": 0.6655772924423218, "sampling/sampling_logp_difference/max": 0.4071004390716553, "sampling/sampling_logp_difference/mean": 0.015878628939390182, "step": 10, "step_time": 27.66670753993094 }, { "clip_ratio/high_max": 0.0007490424031857401, "clip_ratio/high_mean": 0.0007490424031857401, "clip_ratio/low_mean": 0.00011397197667974978, "clip_ratio/low_min": 0.00011397197667974978, "clip_ratio/region_mean": 0.0008630143827758729, "completions/clipped_ratio": 0.47999998927116394, "completions/max_length": 2048.0, "completions/max_terminated_length": 2034.0, "completions/mean_length": 1854.7999267578125, "completions/mean_terminated_length": 1676.4615478515625, "completions/min_length": 1178.0, "completions/min_terminated_length": 1178.0, "entropy": 0.24558066129684447, "epoch": 0.029891304347826088, "frac_reward_zero_std": 0.0, "grad_norm": 0.01811002753674984, "learning_rate": 5e-05, "loss": 0.08611460030078888, "num_tokens": 964445.0, "reward": -0.32566404342651367, "reward_std": 0.5863887667655945, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.9056640863418579, "rewards/length_penalty/std": 0.12008737027645111, "sampling/importance_sampling_ratio/max": 1.515191912651062, "sampling/importance_sampling_ratio/mean": 0.9914591312408447, "sampling/importance_sampling_ratio/min": 0.2949647009372711, "sampling/sampling_logp_difference/max": 1.2208995819091797, "sampling/sampling_logp_difference/mean": 0.0158162210136652, "step": 11, "step_time": 28.245034573134035 }, { "clip_ratio/high_max": 0.0004984279570635408, "clip_ratio/high_mean": 0.0004984279570635408, "clip_ratio/low_mean": 0.00014867318095639347, "clip_ratio/low_min": 0.00014867318095639347, "clip_ratio/region_mean": 0.0006471011089161039, "completions/clipped_ratio": 0.5, "completions/max_length": 2048.0, "completions/max_terminated_length": 2027.0, "completions/mean_length": 1772.43994140625, "completions/mean_terminated_length": 1496.8800048828125, "completions/min_length": 1000.0, "completions/min_terminated_length": 1000.0, "entropy": 0.2841207951307297, "epoch": 0.03260869565217391, "frac_reward_zero_std": 0.0, "grad_norm": 0.018444733694195747, "learning_rate": 5e-05, "loss": 0.10594244301319122, "num_tokens": 1055707.0, "reward": -0.3654492199420929, "reward_std": 0.6480823159217834, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.8654491901397705, "rewards/length_penalty/std": 0.1661682426929474, "sampling/importance_sampling_ratio/max": 1.595076084136963, "sampling/importance_sampling_ratio/mean": 0.9902372360229492, "sampling/importance_sampling_ratio/min": 0.6496405005455017, "sampling/sampling_logp_difference/max": 0.46692144870758057, "sampling/sampling_logp_difference/mean": 0.018158787861466408, "step": 12, "step_time": 28.327091000042856 }, { "clip_ratio/high_max": 0.0005573231639573351, "clip_ratio/high_mean": 0.0005573231639573351, "clip_ratio/low_mean": 4.4240543502382934e-05, "clip_ratio/low_min": 4.4240543502382934e-05, "clip_ratio/region_mean": 0.0006015637016389519, "completions/clipped_ratio": 0.3999999761581421, "completions/max_length": 2048.0, "completions/max_terminated_length": 1992.0, "completions/mean_length": 1732.1199951171875, "completions/mean_terminated_length": 1521.533447265625, "completions/min_length": 840.0, "completions/min_terminated_length": 840.0, "entropy": 0.25946328043937683, "epoch": 0.035326086956521736, "frac_reward_zero_std": 0.0, "grad_norm": 0.01701098121702671, "learning_rate": 5e-05, "loss": 0.07992903143167496, "num_tokens": 1145333.0, "reward": -0.40576171875, "reward_std": 0.6230165958404541, "rewards/correctness/mean": 0.4399999976158142, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.8457617163658142, "rewards/length_penalty/std": 0.18863798677921295, "sampling/importance_sampling_ratio/max": 1.4509283304214478, "sampling/importance_sampling_ratio/mean": 0.9908791780471802, "sampling/importance_sampling_ratio/min": 0.6448545455932617, "sampling/sampling_logp_difference/max": 0.43873047828674316, "sampling/sampling_logp_difference/mean": 0.016880951821804047, "step": 13, "step_time": 27.769306767731905 }, { "clip_ratio/high_max": 0.000875298772007227, "clip_ratio/high_mean": 0.000875298772007227, "clip_ratio/low_mean": 0.00014390671713044868, "clip_ratio/low_min": 0.00014390671713044868, "clip_ratio/region_mean": 0.001019205478951335, "completions/clipped_ratio": 0.47999998927116394, "completions/max_length": 2048.0, "completions/max_terminated_length": 2015.0, "completions/mean_length": 1752.8399658203125, "completions/mean_terminated_length": 1480.3846435546875, "completions/min_length": 975.0, "completions/min_terminated_length": 975.0, "entropy": 0.2478518307209015, "epoch": 0.03804347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.017872340977191925, "learning_rate": 5e-05, "loss": 0.10336586833000183, "num_tokens": 1235535.0, "reward": -0.3358789086341858, "reward_std": 0.654521644115448, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.8558788895606995, "rewards/length_penalty/std": 0.18034636974334717, "sampling/importance_sampling_ratio/max": 1.4315266609191895, "sampling/importance_sampling_ratio/mean": 0.9911905527114868, "sampling/importance_sampling_ratio/min": 0.5119919180870056, "sampling/sampling_logp_difference/max": 0.6694464683532715, "sampling/sampling_logp_difference/mean": 0.016088664531707764, "step": 14, "step_time": 28.357874361332506 }, { "clip_ratio/high_max": 0.0006910582887940109, "clip_ratio/high_mean": 0.0006910582887940109, "clip_ratio/low_mean": 8.928978495532647e-05, "clip_ratio/low_min": 8.928978495532647e-05, "clip_ratio/region_mean": 0.0007803480722941459, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 1903.0, "completions/mean_length": 1541.0399169921875, "completions/mean_terminated_length": 1343.888916015625, "completions/min_length": 961.0, "completions/min_terminated_length": 961.0, "entropy": 0.17789481580257416, "epoch": 0.04076086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.019630515947937965, "learning_rate": 5e-05, "loss": 0.0840536430478096, "num_tokens": 1315167.0, "reward": -0.2124609351158142, "reward_std": 0.5916866064071655, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.7524609565734863, "rewards/length_penalty/std": 0.18993473052978516, "sampling/importance_sampling_ratio/max": 1.4382117986679077, "sampling/importance_sampling_ratio/mean": 0.9938375353813171, "sampling/importance_sampling_ratio/min": 0.6220142841339111, "sampling/sampling_logp_difference/max": 0.4747922420501709, "sampling/sampling_logp_difference/mean": 0.011926891282200813, "step": 15, "step_time": 27.46101871319115 }, { "clip_ratio/high_max": 0.0012169468565844, "clip_ratio/high_mean": 0.0012169468565844, "clip_ratio/low_mean": 6.13222160609439e-05, "clip_ratio/low_min": 6.13222160609439e-05, "clip_ratio/region_mean": 0.0012782690697349607, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 1896.0, "completions/mean_length": 1608.760009765625, "completions/mean_terminated_length": 1290.689697265625, "completions/min_length": 763.0, "completions/min_terminated_length": 763.0, "entropy": 0.25876877307891843, "epoch": 0.043478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.01870792917907238, "learning_rate": 5e-05, "loss": 0.10105059295892715, "num_tokens": 1399705.0, "reward": -0.18552733957767487, "reward_std": 0.6857022643089294, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.7855273485183716, "rewards/length_penalty/std": 0.22407564520835876, "sampling/importance_sampling_ratio/max": 1.8998873233795166, "sampling/importance_sampling_ratio/mean": 0.990841805934906, "sampling/importance_sampling_ratio/min": 0.5615923404693604, "sampling/sampling_logp_difference/max": 0.6417945623397827, "sampling/sampling_logp_difference/mean": 0.016635797917842865, "step": 16, "step_time": 27.157737250905484 }, { "clip_ratio/high_max": 0.0006164561185869388, "clip_ratio/high_mean": 0.0006164561185869388, "clip_ratio/low_mean": 0.0001622869363927748, "clip_ratio/low_min": 0.0001622869363927748, "clip_ratio/region_mean": 0.0007787430338794365, "completions/clipped_ratio": 0.47999998927116394, "completions/max_length": 2048.0, "completions/max_terminated_length": 2028.0, "completions/mean_length": 1708.8399658203125, "completions/mean_terminated_length": 1395.769287109375, "completions/min_length": 868.0, "completions/min_terminated_length": 868.0, "entropy": 0.2281492084264755, "epoch": 0.04619565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.01940748654305935, "learning_rate": 5e-05, "loss": 0.07547423243522644, "num_tokens": 1487797.0, "reward": -0.3143945336341858, "reward_std": 0.6761555075645447, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.8343945145606995, "rewards/length_penalty/std": 0.20066553354263306, "sampling/importance_sampling_ratio/max": 1.667822241783142, "sampling/importance_sampling_ratio/mean": 0.9918660521507263, "sampling/importance_sampling_ratio/min": 0.5456497669219971, "sampling/sampling_logp_difference/max": 0.6057779788970947, "sampling/sampling_logp_difference/mean": 0.01505908090621233, "step": 17, "step_time": 27.55593749205582 }, { "clip_ratio/high_max": 0.0007688433746807277, "clip_ratio/high_mean": 0.0007688433746807277, "clip_ratio/low_mean": 6.974122588871978e-05, "clip_ratio/low_min": 6.974122588871978e-05, "clip_ratio/region_mean": 0.0008385846158489585, "completions/clipped_ratio": 0.23999999463558197, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 1681.239990234375, "completions/mean_terminated_length": 1565.4210205078125, "completions/min_length": 1013.0, "completions/min_terminated_length": 1013.0, "entropy": 0.2069709062576294, "epoch": 0.04891304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.01836375705897808, "learning_rate": 5e-05, "loss": 0.07770835608243942, "num_tokens": 1574359.0, "reward": -0.20091795921325684, "reward_std": 0.6101290583610535, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.8209179639816284, "rewards/length_penalty/std": 0.1540166735649109, "sampling/importance_sampling_ratio/max": 1.4398912191390991, "sampling/importance_sampling_ratio/mean": 0.9928213357925415, "sampling/importance_sampling_ratio/min": 0.6220183372497559, "sampling/sampling_logp_difference/max": 0.4747856855392456, "sampling/sampling_logp_difference/mean": 0.013566325418651104, "step": 18, "step_time": 27.951239524874836 }, { "clip_ratio/high_max": 0.0014653601916506886, "clip_ratio/high_mean": 0.0014653601916506886, "clip_ratio/low_mean": 0.00016244076541624964, "clip_ratio/low_min": 0.00016244076541624964, "clip_ratio/region_mean": 0.0016278009628877045, "completions/clipped_ratio": 0.5600000023841858, "completions/max_length": 2048.0, "completions/max_terminated_length": 2028.0, "completions/mean_length": 1752.8199462890625, "completions/mean_terminated_length": 1377.1363525390625, "completions/min_length": 589.0, "completions/min_terminated_length": 589.0, "entropy": 0.2843230068683624, "epoch": 0.051630434782608696, "frac_reward_zero_std": 0.0, "grad_norm": 0.019417675212025642, "learning_rate": 5e-05, "loss": 0.0901256799697876, "num_tokens": 1665430.0, "reward": -0.3958691358566284, "reward_std": 0.6787852048873901, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.8558691143989563, "rewards/length_penalty/std": 0.22009503841400146, "sampling/importance_sampling_ratio/max": 1.4385100603103638, "sampling/importance_sampling_ratio/mean": 0.9899238348007202, "sampling/importance_sampling_ratio/min": 0.5758324265480042, "sampling/sampling_logp_difference/max": 0.5519386529922485, "sampling/sampling_logp_difference/mean": 0.0180143304169178, "step": 19, "step_time": 28.49813059018925 }, { "clip_ratio/high_max": 0.0012508025160059334, "clip_ratio/high_mean": 0.0012508025160059334, "clip_ratio/low_mean": 5.348440172383562e-05, "clip_ratio/low_min": 5.348440172383562e-05, "clip_ratio/region_mean": 0.0013042868813499808, "completions/clipped_ratio": 0.5199999809265137, "completions/max_length": 2048.0, "completions/max_terminated_length": 2046.0, "completions/mean_length": 1844.43994140625, "completions/mean_terminated_length": 1623.916748046875, "completions/min_length": 1120.0, "completions/min_terminated_length": 1120.0, "entropy": 0.2606961756944656, "epoch": 0.05434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.01755714789032936, "learning_rate": 5e-05, "loss": 0.09018535166978836, "num_tokens": 1760242.0, "reward": -0.42060545086860657, "reward_std": 0.6140408515930176, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.9006054401397705, "rewards/length_penalty/std": 0.12989383935928345, "sampling/importance_sampling_ratio/max": 1.3885498046875, "sampling/importance_sampling_ratio/mean": 0.9910284280776978, "sampling/importance_sampling_ratio/min": 0.4727957248687744, "sampling/sampling_logp_difference/max": 0.7490918636322021, "sampling/sampling_logp_difference/mean": 0.01643785461783409, "step": 20, "step_time": 27.96463798894547 }, { "clip_ratio/high_max": 0.0007326291175559163, "clip_ratio/high_mean": 0.0007326291175559163, "clip_ratio/low_mean": 0.00013991183222969994, "clip_ratio/low_min": 0.00013991183222969994, "clip_ratio/region_mean": 0.0008725409512408078, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 2016.0, "completions/mean_length": 1689.5599365234375, "completions/mean_terminated_length": 1535.94287109375, "completions/min_length": 1011.0, "completions/min_terminated_length": 1011.0, "entropy": 0.2159762352705002, "epoch": 0.057065217391304345, "frac_reward_zero_std": 0.0, "grad_norm": 0.016861431300640106, "learning_rate": 5e-05, "loss": 0.08257238566875458, "num_tokens": 1848130.0, "reward": -0.12498046457767487, "reward_std": 0.5931032299995422, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.8249804973602295, "rewards/length_penalty/std": 0.17415757477283478, "sampling/importance_sampling_ratio/max": 1.4638551473617554, "sampling/importance_sampling_ratio/mean": 0.9920001029968262, "sampling/importance_sampling_ratio/min": 0.5236321091651917, "sampling/sampling_logp_difference/max": 0.6469658613204956, "sampling/sampling_logp_difference/mean": 0.014182405546307564, "step": 21, "step_time": 26.797411655075848 }, { "clip_ratio/high_max": 0.0011293109448160977, "clip_ratio/high_mean": 0.0011293109448160977, "clip_ratio/low_mean": 9.711003804113716e-05, "clip_ratio/low_min": 9.711003804113716e-05, "clip_ratio/region_mean": 0.0012264209799468517, "completions/clipped_ratio": 0.2800000011920929, "completions/max_length": 2048.0, "completions/max_terminated_length": 1946.0, "completions/mean_length": 1501.0599365234375, "completions/mean_terminated_length": 1288.361083984375, "completions/min_length": 687.0, "completions/min_terminated_length": 687.0, "entropy": 0.2569431811571121, "epoch": 0.059782608695652176, "frac_reward_zero_std": 0.0, "grad_norm": 0.020697707310318947, "learning_rate": 5e-05, "loss": 0.12516915798187256, "num_tokens": 1928253.0, "reward": 0.0070605468936264515, "reward_std": 0.6210074424743652, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.7329394817352295, "rewards/length_penalty/std": 0.21821558475494385, "sampling/importance_sampling_ratio/max": 1.6245068311691284, "sampling/importance_sampling_ratio/mean": 0.9910734295845032, "sampling/importance_sampling_ratio/min": 0.5689723491668701, "sampling/sampling_logp_difference/max": 0.5639233589172363, "sampling/sampling_logp_difference/mean": 0.016499562188982964, "step": 22, "step_time": 27.23643856518902 }, { "clip_ratio/high_max": 0.0009234562981873751, "clip_ratio/high_mean": 0.0009234562981873751, "clip_ratio/low_mean": 0.00011172640952281654, "clip_ratio/low_min": 0.00011172640952281654, "clip_ratio/region_mean": 0.0010351826902478933, "completions/clipped_ratio": 0.3400000035762787, "completions/max_length": 2048.0, "completions/max_terminated_length": 1951.0, "completions/mean_length": 1656.260009765625, "completions/mean_terminated_length": 1454.45458984375, "completions/min_length": 875.0, "completions/min_terminated_length": 875.0, "entropy": 0.2721762388944626, "epoch": 0.0625, "frac_reward_zero_std": 0.0, "grad_norm": 0.02058061957359314, "learning_rate": 5e-05, "loss": 0.1155262216925621, "num_tokens": 2017186.0, "reward": -0.2487206906080246, "reward_std": 0.6356157064437866, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.8087207078933716, "rewards/length_penalty/std": 0.1684350073337555, "sampling/importance_sampling_ratio/max": 1.5564347505569458, "sampling/importance_sampling_ratio/mean": 0.9903766512870789, "sampling/importance_sampling_ratio/min": 0.5593669414520264, "sampling/sampling_logp_difference/max": 0.5809495449066162, "sampling/sampling_logp_difference/mean": 0.01754668541252613, "step": 23, "step_time": 27.693883403902873 }, { "clip_ratio/high_max": 0.0005631370135233737, "clip_ratio/high_mean": 0.0005631370135233737, "clip_ratio/low_mean": 0.00010890306512010284, "clip_ratio/low_min": 0.00010890306512010284, "clip_ratio/region_mean": 0.0006720400706399232, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 2034.0, "completions/mean_length": 1659.1199951171875, "completions/mean_terminated_length": 1377.5172119140625, "completions/min_length": 734.0, "completions/min_terminated_length": 734.0, "entropy": 0.2400749534368515, "epoch": 0.06521739130434782, "frac_reward_zero_std": 0.0, "grad_norm": 0.017921488732099533, "learning_rate": 5e-05, "loss": 0.0665205717086792, "num_tokens": 2102432.0, "reward": -0.2701171934604645, "reward_std": 0.675529420375824, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.8101171851158142, "rewards/length_penalty/std": 0.22776594758033752, "sampling/importance_sampling_ratio/max": 1.6843202114105225, "sampling/importance_sampling_ratio/mean": 0.9915715456008911, "sampling/importance_sampling_ratio/min": 0.4471953809261322, "sampling/sampling_logp_difference/max": 0.8047597408294678, "sampling/sampling_logp_difference/mean": 0.01562676392495632, "step": 24, "step_time": 27.030204062117264 }, { "clip_ratio/high_max": 0.0010878795757889748, "clip_ratio/high_mean": 0.0010878795757889748, "clip_ratio/low_mean": 0.00010777307761600241, "clip_ratio/low_min": 0.00010777307761600241, "clip_ratio/region_mean": 0.0011956526315771044, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 2004.0, "completions/mean_length": 1457.0599365234375, "completions/mean_terminated_length": 1290.3846435546875, "completions/min_length": 754.0, "completions/min_terminated_length": 754.0, "entropy": 0.28386659622192384, "epoch": 0.06793478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.021549172699451447, "learning_rate": 5e-05, "loss": 0.10626962780952454, "num_tokens": 2177685.0, "reward": 0.04854492098093033, "reward_std": 0.6079964637756348, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.7114551067352295, "rewards/length_penalty/std": 0.21497680246829987, "sampling/importance_sampling_ratio/max": 1.7750171422958374, "sampling/importance_sampling_ratio/mean": 0.9901482462882996, "sampling/importance_sampling_ratio/min": 0.3599497675895691, "sampling/sampling_logp_difference/max": 1.021790862083435, "sampling/sampling_logp_difference/mean": 0.018604418262839317, "step": 25, "step_time": 25.64425123319961 }, { "clip_ratio/high_max": 0.000790244567906484, "clip_ratio/high_mean": 0.000790244567906484, "clip_ratio/low_mean": 4.310927906772122e-05, "clip_ratio/low_min": 4.310927906772122e-05, "clip_ratio/region_mean": 0.0008333538484293967, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 2031.0, "completions/mean_length": 1346.1199951171875, "completions/mean_terminated_length": 1285.0870361328125, "completions/min_length": 729.0, "completions/min_terminated_length": 729.0, "entropy": 0.2296551436185837, "epoch": 0.07065217391304347, "frac_reward_zero_std": 0.0, "grad_norm": 0.020984938368201256, "learning_rate": 5e-05, "loss": 0.09681613743305206, "num_tokens": 2247461.0, "reward": -0.09728515148162842, "reward_std": 0.5121884346008301, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.6572851538658142, "rewards/length_penalty/std": 0.19092224538326263, "sampling/importance_sampling_ratio/max": 1.9684982299804688, "sampling/importance_sampling_ratio/mean": 0.9919283986091614, "sampling/importance_sampling_ratio/min": 0.34581977128982544, "sampling/sampling_logp_difference/max": 1.0618375539779663, "sampling/sampling_logp_difference/mean": 0.015773361548781395, "step": 26, "step_time": 25.79908507759683 }, { "clip_ratio/high_max": 0.0005374234810005873, "clip_ratio/high_mean": 0.0005374234810005873, "clip_ratio/low_mean": 0.00010813681001309305, "clip_ratio/low_min": 0.00010813681001309305, "clip_ratio/region_mean": 0.0006455602881032974, "completions/clipped_ratio": 0.3999999761581421, "completions/max_length": 2048.0, "completions/max_terminated_length": 1926.0, "completions/mean_length": 1517.8599853515625, "completions/mean_terminated_length": 1164.433349609375, "completions/min_length": 606.0, "completions/min_terminated_length": 606.0, "entropy": 0.2029076486825943, "epoch": 0.07336956521739131, "frac_reward_zero_std": 0.0, "grad_norm": 0.017958836629986763, "learning_rate": 5e-05, "loss": 0.028417475521564484, "num_tokens": 2325604.0, "reward": -0.12114257365465164, "reward_std": 0.709344208240509, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.7411425709724426, "rewards/length_penalty/std": 0.2490270882844925, "sampling/importance_sampling_ratio/max": 1.7574549913406372, "sampling/importance_sampling_ratio/mean": 0.9929352402687073, "sampling/importance_sampling_ratio/min": 0.2425677329301834, "sampling/sampling_logp_difference/max": 1.4164743423461914, "sampling/sampling_logp_difference/mean": 0.013834730722010136, "step": 27, "step_time": 25.780867578927428 }, { "clip_ratio/high_max": 0.0008416285156272352, "clip_ratio/high_mean": 0.0008416285156272352, "clip_ratio/low_mean": 4.83594267279841e-05, "clip_ratio/low_min": 4.83594267279841e-05, "clip_ratio/region_mean": 0.0008899879525415599, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 1903.0, "completions/mean_length": 1254.239990234375, "completions/mean_terminated_length": 1203.574462890625, "completions/min_length": 796.0, "completions/min_terminated_length": 796.0, "entropy": 0.18921910226345062, "epoch": 0.07608695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.019402744248509407, "learning_rate": 5e-05, "loss": 0.108873650431633, "num_tokens": 2391676.0, "reward": 0.12757812440395355, "reward_std": 0.5219429731369019, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308748841285706, "rewards/length_penalty/mean": -0.6124218702316284, "rewards/length_penalty/std": 0.16508276760578156, "sampling/importance_sampling_ratio/max": 1.7898136377334595, "sampling/importance_sampling_ratio/mean": 0.9934007525444031, "sampling/importance_sampling_ratio/min": 0.2999454140663147, "sampling/sampling_logp_difference/max": 1.2041547298431396, "sampling/sampling_logp_difference/mean": 0.013383659534156322, "step": 28, "step_time": 25.343337242724374 }, { "clip_ratio/high_max": 0.0007759740750771016, "clip_ratio/high_mean": 0.0007759740750771016, "clip_ratio/low_mean": 8.760135315242224e-05, "clip_ratio/low_min": 8.760135315242224e-05, "clip_ratio/region_mean": 0.0008635754231363535, "completions/clipped_ratio": 0.3799999952316284, "completions/max_length": 2048.0, "completions/max_terminated_length": 2005.0, "completions/mean_length": 1616.659912109375, "completions/mean_terminated_length": 1352.290283203125, "completions/min_length": 575.0, "completions/min_terminated_length": 575.0, "entropy": 0.24235570728778838, "epoch": 0.07880434782608696, "frac_reward_zero_std": 0.0, "grad_norm": 0.019807059317827225, "learning_rate": 5e-05, "loss": 0.06528324633836746, "num_tokens": 2475949.0, "reward": -0.12938475608825684, "reward_std": 0.6638873219490051, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.47851812839508057, "rewards/length_penalty/mean": -0.7893847823143005, "rewards/length_penalty/std": 0.25617480278015137, "sampling/importance_sampling_ratio/max": 1.7938244342803955, "sampling/importance_sampling_ratio/mean": 0.9914634823799133, "sampling/importance_sampling_ratio/min": 0.3581945598125458, "sampling/sampling_logp_difference/max": 1.0266790390014648, "sampling/sampling_logp_difference/mean": 0.01648593135178089, "step": 29, "step_time": 26.831963051110506 }, { "clip_ratio/high_max": 0.0007342056138440967, "clip_ratio/high_mean": 0.0007342056138440967, "clip_ratio/low_mean": 6.386294553522021e-05, "clip_ratio/low_min": 6.386294553522021e-05, "clip_ratio/region_mean": 0.000798068568110466, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1963.0, "completions/mean_length": 1230.659912109375, "completions/mean_terminated_length": 1139.844482421875, "completions/min_length": 761.0, "completions/min_terminated_length": 761.0, "entropy": 0.1697321891784668, "epoch": 0.08152173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.015965549275279045, "learning_rate": 5e-05, "loss": 0.07354339957237244, "num_tokens": 2539272.0, "reward": 0.319091796875, "reward_std": 0.4232766628265381, "rewards/correctness/mean": 0.9200000166893005, "rewards/correctness/std": 0.27404749393463135, "rewards/length_penalty/mean": -0.6009082198143005, "rewards/length_penalty/std": 0.19723902642726898, "sampling/importance_sampling_ratio/max": 1.9091259241104126, "sampling/importance_sampling_ratio/mean": 0.9938931465148926, "sampling/importance_sampling_ratio/min": 0.006599857937544584, "sampling/sampling_logp_difference/max": 5.020707130432129, "sampling/sampling_logp_difference/mean": 0.012903784401714802, "step": 30, "step_time": 24.605497697601095 }, { "clip_ratio/high_max": 0.0005373104766476899, "clip_ratio/high_mean": 0.0005373104766476899, "clip_ratio/low_mean": 0.00013891156122554095, "clip_ratio/low_min": 0.00013891156122554095, "clip_ratio/region_mean": 0.0006762220233213157, "completions/clipped_ratio": 0.3400000035762787, "completions/max_length": 2048.0, "completions/max_terminated_length": 1972.0, "completions/mean_length": 1636.5999755859375, "completions/mean_terminated_length": 1424.666748046875, "completions/min_length": 671.0, "completions/min_terminated_length": 671.0, "entropy": 0.18908907771110534, "epoch": 0.08423913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.01748133823275566, "learning_rate": 5e-05, "loss": 0.05618336424231529, "num_tokens": 2624232.0, "reward": -0.4791210889816284, "reward_std": 0.46296438574790955, "rewards/correctness/mean": 0.3199999928474426, "rewards/correctness/std": 0.4712120592594147, "rewards/length_penalty/mean": -0.799121081829071, "rewards/length_penalty/std": 0.2133444994688034, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.993071436882019, "sampling/importance_sampling_ratio/min": 0.3394291400909424, "sampling/sampling_logp_difference/max": 1.2877426147460938, "sampling/sampling_logp_difference/mean": 0.013400268740952015, "step": 31, "step_time": 27.240520113147795 }, { "clip_ratio/high_max": 0.0011355961789377034, "clip_ratio/high_mean": 0.0011355961789377034, "clip_ratio/low_mean": 0.00015650656423531472, "clip_ratio/low_min": 0.00015650656423531472, "clip_ratio/region_mean": 0.0012921027606353163, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 2022.0, "completions/mean_length": 1262.5599365234375, "completions/mean_terminated_length": 1155.45458984375, "completions/min_length": 595.0, "completions/min_terminated_length": 595.0, "entropy": 0.22945527732372284, "epoch": 0.08695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.01773648150265217, "learning_rate": 5e-05, "loss": 0.09414535760879517, "num_tokens": 2689290.0, "reward": 0.2235156148672104, "reward_std": 0.5308910012245178, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.6164844036102295, "rewards/length_penalty/std": 0.2093491554260254, "sampling/importance_sampling_ratio/max": 2.3375766277313232, "sampling/importance_sampling_ratio/mean": 0.9917176961898804, "sampling/importance_sampling_ratio/min": 0.30112117528915405, "sampling/sampling_logp_difference/max": 1.200242519378662, "sampling/sampling_logp_difference/mean": 0.01688900962471962, "step": 32, "step_time": 24.735927454195917 }, { "clip_ratio/high_max": 0.000496267294511199, "clip_ratio/high_mean": 0.000496267294511199, "clip_ratio/low_mean": 0.00010696919343899935, "clip_ratio/low_min": 0.00010696919343899935, "clip_ratio/region_mean": 0.0006032364792190493, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1985.0, "completions/mean_length": 1416.780029296875, "completions/mean_terminated_length": 1195.0, "completions/min_length": 765.0, "completions/min_terminated_length": 765.0, "entropy": 0.19317569732666015, "epoch": 0.08967391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.01730223558843136, "learning_rate": 5e-05, "loss": 0.029047608375549316, "num_tokens": 2763769.0, "reward": -0.09178710728883743, "reward_std": 0.6523070335388184, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716163635254, "rewards/length_penalty/mean": -0.6917871236801147, "rewards/length_penalty/std": 0.2441810518503189, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930633902549744, "sampling/importance_sampling_ratio/min": 0.23537996411323547, "sampling/sampling_logp_difference/max": 1.446554183959961, "sampling/sampling_logp_difference/mean": 0.014113553799688816, "step": 33, "step_time": 26.25649970700033 }, { "clip_ratio/high_max": 0.0010051162447780372, "clip_ratio/high_mean": 0.0010051162447780372, "clip_ratio/low_mean": 7.109031575964763e-05, "clip_ratio/low_min": 7.109031575964763e-05, "clip_ratio/region_mean": 0.0010762065765447914, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 1776.0, "completions/mean_length": 1481.6400146484375, "completions/mean_terminated_length": 1238.914306640625, "completions/min_length": 812.0, "completions/min_terminated_length": 812.0, "entropy": 0.23732842803001403, "epoch": 0.09239130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.020243240520358086, "learning_rate": 5e-05, "loss": 0.11132572591304779, "num_tokens": 2841741.0, "reward": -0.04345703125, "reward_std": 0.6613745093345642, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.4712120592594147, "rewards/length_penalty/mean": -0.7234570384025574, "rewards/length_penalty/std": 0.21751287579536438, "sampling/importance_sampling_ratio/max": 2.625032424926758, "sampling/importance_sampling_ratio/mean": 0.9916952252388, "sampling/importance_sampling_ratio/min": 0.20833513140678406, "sampling/sampling_logp_difference/max": 1.568607211112976, "sampling/sampling_logp_difference/mean": 0.01689102314412594, "step": 34, "step_time": 26.093267522752285 }, { "clip_ratio/high_max": 0.0007707345008384437, "clip_ratio/high_mean": 0.0007707345008384437, "clip_ratio/low_mean": 5.561825819313526e-05, "clip_ratio/low_min": 5.561825819313526e-05, "clip_ratio/region_mean": 0.0008263527532108128, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1967.0, "completions/mean_length": 1388.7999267578125, "completions/mean_terminated_length": 1157.189208984375, "completions/min_length": 717.0, "completions/min_terminated_length": 717.0, "entropy": 0.25725961923599244, "epoch": 0.09510869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.020236779004335403, "learning_rate": 5e-05, "loss": 0.10550195723772049, "num_tokens": 2913911.0, "reward": 0.06187499687075615, "reward_std": 0.646725058555603, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.6781250238418579, "rewards/length_penalty/std": 0.22613263130187988, "sampling/importance_sampling_ratio/max": 2.0787932872772217, "sampling/importance_sampling_ratio/mean": 0.990626335144043, "sampling/importance_sampling_ratio/min": 0.1848619133234024, "sampling/sampling_logp_difference/max": 1.6881461143493652, "sampling/sampling_logp_difference/mean": 0.018169380724430084, "step": 35, "step_time": 25.364839713089168 }, { "clip_ratio/high_max": 0.0006121197395259515, "clip_ratio/high_mean": 0.0006121197395259515, "clip_ratio/low_mean": 0.00011545439338078722, "clip_ratio/low_min": 0.00011545439338078722, "clip_ratio/region_mean": 0.0007275741430930793, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 2018.0, "completions/mean_length": 1189.8399658203125, "completions/mean_terminated_length": 1072.8182373046875, "completions/min_length": 626.0, "completions/min_terminated_length": 626.0, "entropy": 0.20621423423290253, "epoch": 0.09782608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.017101073637604713, "learning_rate": 5e-05, "loss": 0.08683709055185318, "num_tokens": 2975873.0, "reward": 0.2190234363079071, "reward_std": 0.609960675239563, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5809765458106995, "rewards/length_penalty/std": 0.22153306007385254, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.992645263671875, "sampling/importance_sampling_ratio/min": 0.14949937164783478, "sampling/sampling_logp_difference/max": 1.9004631042480469, "sampling/sampling_logp_difference/mean": 0.01638173498213291, "step": 36, "step_time": 25.151533517986536 }, { "clip_ratio/high_max": 0.0017051160102710128, "clip_ratio/high_mean": 0.0017051160102710128, "clip_ratio/low_mean": 4.560533707262948e-05, "clip_ratio/low_min": 4.560533707262948e-05, "clip_ratio/region_mean": 0.001750721363350749, "completions/clipped_ratio": 0.23999999463558197, "completions/max_length": 2048.0, "completions/max_terminated_length": 1696.0, "completions/mean_length": 1309.4599609375, "completions/mean_terminated_length": 1076.23681640625, "completions/min_length": 674.0, "completions/min_terminated_length": 674.0, "entropy": 0.2457641303539276, "epoch": 0.10054347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.01708856225013733, "learning_rate": 5e-05, "loss": 0.08991742134094238, "num_tokens": 3044866.0, "reward": -0.07938476651906967, "reward_std": 0.6378496289253235, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.6393847465515137, "rewards/length_penalty/std": 0.23324140906333923, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9911713600158691, "sampling/importance_sampling_ratio/min": 0.18068967759609222, "sampling/sampling_logp_difference/max": 1.7109742164611816, "sampling/sampling_logp_difference/mean": 0.01840140111744404, "step": 37, "step_time": 25.540588906034827 }, { "clip_ratio/high_max": 0.0007167195668444038, "clip_ratio/high_mean": 0.0007167195668444038, "clip_ratio/low_mean": 7.254186493810266e-05, "clip_ratio/low_min": 7.254186493810266e-05, "clip_ratio/region_mean": 0.0007892614346928894, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1988.0, "completions/mean_length": 1333.4000244140625, "completions/mean_terminated_length": 1235.95458984375, "completions/min_length": 690.0, "completions/min_terminated_length": 690.0, "entropy": 0.17886880338191985, "epoch": 0.10326086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.02032496966421604, "learning_rate": 5e-05, "loss": 0.08079551160335541, "num_tokens": 3114926.0, "reward": 0.16892577707767487, "reward_std": 0.5259281396865845, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.651074230670929, "rewards/length_penalty/std": 0.23055696487426758, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9937843680381775, "sampling/importance_sampling_ratio/min": 0.2627275884151459, "sampling/sampling_logp_difference/max": 1.3366376161575317, "sampling/sampling_logp_difference/mean": 0.013650714419782162, "step": 38, "step_time": 26.08059739181772 }, { "clip_ratio/high_max": 0.0005097337998449802, "clip_ratio/high_mean": 0.0005097337998449802, "clip_ratio/low_mean": 6.99934986187145e-05, "clip_ratio/low_min": 6.99934986187145e-05, "clip_ratio/region_mean": 0.0005797273013740778, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 1743.0, "completions/mean_length": 1207.47998046875, "completions/mean_terminated_length": 970.4102783203125, "completions/min_length": 488.0, "completions/min_terminated_length": 488.0, "entropy": 0.18856219053268433, "epoch": 0.10597826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.020054886117577553, "learning_rate": 5e-05, "loss": 0.0812826082110405, "num_tokens": 3177790.0, "reward": -0.009589843451976776, "reward_std": 0.7438336610794067, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.5895898342132568, "rewards/length_penalty/std": 0.26728230714797974, "sampling/importance_sampling_ratio/max": 2.8230764865875244, "sampling/importance_sampling_ratio/mean": 0.9934998750686646, "sampling/importance_sampling_ratio/min": 0.23803837597370148, "sampling/sampling_logp_difference/max": 1.4353233575820923, "sampling/sampling_logp_difference/mean": 0.01414400339126587, "step": 39, "step_time": 24.726267321035266 }, { "clip_ratio/high_max": 0.0008135152223985642, "clip_ratio/high_mean": 0.0008135152223985642, "clip_ratio/low_mean": 8.447097934549674e-05, "clip_ratio/low_min": 8.447097934549674e-05, "clip_ratio/region_mean": 0.0008979861973784864, "completions/clipped_ratio": 0.0, "completions/max_length": 1624.0, "completions/max_terminated_length": 1624.0, "completions/mean_length": 933.5399780273438, "completions/mean_terminated_length": 933.5399780273438, "completions/min_length": 641.0, "completions/min_terminated_length": 641.0, "entropy": 0.192103710770607, "epoch": 0.10869565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.014751175418496132, "learning_rate": 5e-05, "loss": 0.07916796952486038, "num_tokens": 3227187.0, "reward": 0.5441699028015137, "reward_std": 0.11200839281082153, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.45583006739616394, "rewards/length_penalty/std": 0.11200839281082153, "sampling/importance_sampling_ratio/max": 2.518812656402588, "sampling/importance_sampling_ratio/mean": 0.9928264021873474, "sampling/importance_sampling_ratio/min": 0.1412581503391266, "sampling/sampling_logp_difference/max": 1.9571661949157715, "sampling/sampling_logp_difference/mean": 0.016277069225907326, "step": 40, "step_time": 19.264488755259663 }, { "clip_ratio/high_max": 0.0006308750482276082, "clip_ratio/high_mean": 0.0006308750482276082, "clip_ratio/low_mean": 7.04108999343589e-05, "clip_ratio/low_min": 7.04108999343589e-05, "clip_ratio/region_mean": 0.0007012859452515841, "completions/clipped_ratio": 0.17999999225139618, "completions/max_length": 2048.0, "completions/max_terminated_length": 1934.0, "completions/mean_length": 1309.0999755859375, "completions/mean_terminated_length": 1146.90234375, "completions/min_length": 592.0, "completions/min_terminated_length": 592.0, "entropy": 0.21561425626277925, "epoch": 0.11141304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.020088374614715576, "learning_rate": 5e-05, "loss": 0.08004692196846008, "num_tokens": 3296382.0, "reward": -0.13920898735523224, "reward_std": 0.6492041945457458, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.639208972454071, "rewards/length_penalty/std": 0.229248046875, "sampling/importance_sampling_ratio/max": 2.644029140472412, "sampling/importance_sampling_ratio/mean": 0.9920929670333862, "sampling/importance_sampling_ratio/min": 0.07542523741722107, "sampling/sampling_logp_difference/max": 2.58461332321167, "sampling/sampling_logp_difference/mean": 0.016478460282087326, "step": 41, "step_time": 26.089902371168137 }, { "clip_ratio/high_max": 0.0007611454464495182, "clip_ratio/high_mean": 0.0007611454464495182, "clip_ratio/low_mean": 8.782477234490216e-05, "clip_ratio/low_min": 8.782477234490216e-05, "clip_ratio/region_mean": 0.0008489702187944204, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 1938.0, "completions/mean_length": 1232.4000244140625, "completions/mean_terminated_length": 1077.047607421875, "completions/min_length": 473.0, "completions/min_terminated_length": 473.0, "entropy": 0.19867940843105317, "epoch": 0.11413043478260869, "frac_reward_zero_std": 0.0, "grad_norm": 0.01986713334918022, "learning_rate": 5e-05, "loss": 0.06985890865325928, "num_tokens": 3360992.0, "reward": 0.23824217915534973, "reward_std": 0.5804083347320557, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.601757824420929, "rewards/length_penalty/std": 0.2639937698841095, "sampling/importance_sampling_ratio/max": 2.2874951362609863, "sampling/importance_sampling_ratio/mean": 0.9928309917449951, "sampling/importance_sampling_ratio/min": 0.11545022577047348, "sampling/sampling_logp_difference/max": 2.1589157581329346, "sampling/sampling_logp_difference/mean": 0.015996571630239487, "step": 42, "step_time": 24.868821669137105 }, { "clip_ratio/high_max": 0.000910443312022835, "clip_ratio/high_mean": 0.000910443312022835, "clip_ratio/low_mean": 6.609660922549666e-05, "clip_ratio/low_min": 6.609660922549666e-05, "clip_ratio/region_mean": 0.0009765399037860334, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1646.0, "completions/mean_length": 888.1199951171875, "completions/mean_terminated_length": 864.448974609375, "completions/min_length": 459.0, "completions/min_terminated_length": 459.0, "entropy": 0.21161480844020844, "epoch": 0.11684782608695653, "frac_reward_zero_std": 0.0, "grad_norm": 0.020094897598028183, "learning_rate": 5e-05, "loss": 0.06470935791730881, "num_tokens": 3407368.0, "reward": 0.48634764552116394, "reward_std": 0.3447928726673126, "rewards/correctness/mean": 0.9200000166893005, "rewards/correctness/std": 0.27404749393463135, "rewards/length_penalty/mean": -0.4336523413658142, "rewards/length_penalty/std": 0.17656175792217255, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9923710227012634, "sampling/importance_sampling_ratio/min": 0.11419941484928131, "sampling/sampling_logp_difference/max": 2.169809103012085, "sampling/sampling_logp_difference/mean": 0.018651597201824188, "step": 43, "step_time": 22.883227334124967 }, { "clip_ratio/high_max": 0.0008881673566065729, "clip_ratio/high_mean": 0.0008881673566065729, "clip_ratio/low_mean": 0.00012054646213073283, "clip_ratio/low_min": 0.00012054646213073283, "clip_ratio/region_mean": 0.001008713839109987, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 1955.0, "completions/mean_length": 1177.280029296875, "completions/mean_terminated_length": 1035.534912109375, "completions/min_length": 586.0, "completions/min_terminated_length": 586.0, "entropy": 0.2084593027830124, "epoch": 0.11956521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.019616451114416122, "learning_rate": 5e-05, "loss": 0.12185367941856384, "num_tokens": 3469232.0, "reward": 0.28515625, "reward_std": 0.5443253517150879, "rewards/correctness/mean": 0.8600000143051147, "rewards/correctness/std": 0.3505098223686218, "rewards/length_penalty/mean": -0.5748437643051147, "rewards/length_penalty/std": 0.22794675827026367, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9923920035362244, "sampling/importance_sampling_ratio/min": 0.1075812503695488, "sampling/sampling_logp_difference/max": 2.229508876800537, "sampling/sampling_logp_difference/mean": 0.0175347737967968, "step": 44, "step_time": 25.075815413612872 }, { "clip_ratio/high_max": 0.0009058942960109562, "clip_ratio/high_mean": 0.0009058942960109562, "clip_ratio/low_mean": 7.237063837237657e-05, "clip_ratio/low_min": 7.237063837237657e-05, "clip_ratio/region_mean": 0.0009782649285625666, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1153.760009765625, "completions/mean_terminated_length": 983.4285888671875, "completions/min_length": 451.0, "completions/min_terminated_length": 451.0, "entropy": 0.23205784261226653, "epoch": 0.12228260869565218, "frac_reward_zero_std": 0.0, "grad_norm": 0.021139081567525864, "learning_rate": 5e-05, "loss": 0.06741262972354889, "num_tokens": 3529860.0, "reward": 0.15664061903953552, "reward_std": 0.6854303479194641, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.5633593797683716, "rewards/length_penalty/std": 0.2703804671764374, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9912254214286804, "sampling/importance_sampling_ratio/min": 0.08344912528991699, "sampling/sampling_logp_difference/max": 2.483518123626709, "sampling/sampling_logp_difference/mean": 0.01880672574043274, "step": 45, "step_time": 24.49455920001492 }, { "clip_ratio/high_max": 0.0015542090637609362, "clip_ratio/high_mean": 0.0015542090637609362, "clip_ratio/low_mean": 8.50872034789063e-05, "clip_ratio/low_min": 8.50872034789063e-05, "clip_ratio/region_mean": 0.0016392962774261833, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1445.0, "completions/mean_length": 1179.5999755859375, "completions/mean_terminated_length": 962.5, "completions/min_length": 551.0, "completions/min_terminated_length": 551.0, "entropy": 0.23213451504707336, "epoch": 0.125, "frac_reward_zero_std": 0.0, "grad_norm": 0.016054287552833557, "learning_rate": 5e-05, "loss": 0.054812848567962646, "num_tokens": 3592350.0, "reward": 0.22402343153953552, "reward_std": 0.6261232495307922, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.575976550579071, "rewards/length_penalty/std": 0.23599623143672943, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.991656482219696, "sampling/importance_sampling_ratio/min": 0.0905810073018074, "sampling/sampling_logp_difference/max": 2.401510715484619, "sampling/sampling_logp_difference/mean": 0.018099654465913773, "step": 46, "step_time": 24.68149675289169 }, { "clip_ratio/high_max": 0.0008501063100993633, "clip_ratio/high_mean": 0.0008501063100993633, "clip_ratio/low_mean": 0.00010414896241854877, "clip_ratio/low_min": 0.00010414896241854877, "clip_ratio/region_mean": 0.0009542552754282952, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1882.0, "completions/mean_length": 944.0199584960938, "completions/mean_terminated_length": 898.0208740234375, "completions/min_length": 576.0, "completions/min_terminated_length": 576.0, "entropy": 0.19239770770072936, "epoch": 0.12771739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.020118683576583862, "learning_rate": 5e-05, "loss": 0.035656265914440155, "num_tokens": 3641941.0, "reward": 0.4390527307987213, "reward_std": 0.44222262501716614, "rewards/correctness/mean": 0.8999999761581421, "rewards/correctness/std": 0.30304577946662903, "rewards/length_penalty/mean": -0.46094727516174316, "rewards/length_penalty/std": 0.1884309947490692, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9925170540809631, "sampling/importance_sampling_ratio/min": 0.10729321092367172, "sampling/sampling_logp_difference/max": 2.232189893722534, "sampling/sampling_logp_difference/mean": 0.01707930490374565, "step": 47, "step_time": 23.979559221770614 }, { "clip_ratio/high_max": 0.0007360532443271949, "clip_ratio/high_mean": 0.0007360532443271949, "clip_ratio/low_mean": 5.149670469108969e-05, "clip_ratio/low_min": 5.149670469108969e-05, "clip_ratio/region_mean": 0.0007875499461079017, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 2012.0, "completions/mean_length": 1140.6199951171875, "completions/mean_terminated_length": 1039.800048828125, "completions/min_length": 514.0, "completions/min_terminated_length": 514.0, "entropy": 0.18729614913463594, "epoch": 0.13043478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.019802836701273918, "learning_rate": 5e-05, "loss": 0.06913310289382935, "num_tokens": 3701932.0, "reward": 0.12305663526058197, "reward_std": 0.6152191162109375, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.5569433569908142, "rewards/length_penalty/std": 0.24529792368412018, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9927173256874084, "sampling/importance_sampling_ratio/min": 0.13456226885318756, "sampling/sampling_logp_difference/max": 2.005728244781494, "sampling/sampling_logp_difference/mean": 0.016090555116534233, "step": 48, "step_time": 24.360856070183218 }, { "clip_ratio/high_max": 0.0014991959935287014, "clip_ratio/high_mean": 0.0014991959935287014, "clip_ratio/low_mean": 5.690939142368734e-05, "clip_ratio/low_min": 5.690939142368734e-05, "clip_ratio/region_mean": 0.001556105344207026, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1192.0, "completions/mean_length": 1090.3800048828125, "completions/mean_terminated_length": 850.9750366210938, "completions/min_length": 518.0, "completions/min_terminated_length": 518.0, "entropy": 0.23595961928367615, "epoch": 0.1331521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.015036417171359062, "learning_rate": 5e-05, "loss": 0.054190024733543396, "num_tokens": 3759141.0, "reward": 0.2675878703594208, "reward_std": 0.6470195651054382, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.5324121117591858, "rewards/length_penalty/std": 0.2540016770362854, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9912841320037842, "sampling/importance_sampling_ratio/min": 0.12618088722229004, "sampling/sampling_logp_difference/max": 2.0700387954711914, "sampling/sampling_logp_difference/mean": 0.019528958946466446, "step": 49, "step_time": 24.133017131127417 }, { "clip_ratio/high_max": 0.0015399554860778153, "clip_ratio/high_mean": 0.0015399554860778153, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015399554860778153, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 981.9400024414062, "completions/mean_terminated_length": 715.4249877929688, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.2847625881433487, "epoch": 0.1358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.01285274513065815, "learning_rate": 5e-05, "loss": 0.03275208920240402, "num_tokens": 3810648.0, "reward": 0.12053710967302322, "reward_std": 0.7175264954566956, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.4794628918170929, "rewards/length_penalty/std": 0.27745550870895386, "sampling/importance_sampling_ratio/max": 2.4426708221435547, "sampling/importance_sampling_ratio/mean": 0.9890289306640625, "sampling/importance_sampling_ratio/min": 0.10526978224515915, "sampling/sampling_logp_difference/max": 2.2512288093566895, "sampling/sampling_logp_difference/mean": 0.022413378581404686, "step": 50, "step_time": 24.217485108412802 }, { "clip_ratio/high_max": 0.0006860518013127148, "clip_ratio/high_mean": 0.0006860518013127148, "clip_ratio/low_mean": 6.418603006750346e-05, "clip_ratio/low_min": 6.418603006750346e-05, "clip_ratio/region_mean": 0.0007502378197386861, "completions/clipped_ratio": 0.0, "completions/max_length": 1852.0, "completions/max_terminated_length": 1852.0, "completions/mean_length": 936.3200073242188, "completions/mean_terminated_length": 936.3200073242188, "completions/min_length": 428.0, "completions/min_terminated_length": 428.0, "entropy": 0.19295797646045684, "epoch": 0.13858695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.016714785248041153, "learning_rate": 5e-05, "loss": 0.08077684044837952, "num_tokens": 3859974.0, "reward": 0.5428124666213989, "reward_std": 0.17791098356246948, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.4571875035762787, "rewards/length_penalty/std": 0.17791098356246948, "sampling/importance_sampling_ratio/max": 2.891409397125244, "sampling/importance_sampling_ratio/mean": 0.992777943611145, "sampling/importance_sampling_ratio/min": 0.12538689374923706, "sampling/sampling_logp_difference/max": 2.0763511657714844, "sampling/sampling_logp_difference/mean": 0.017098676413297653, "step": 51, "step_time": 21.442696853075176 }, { "clip_ratio/high_max": 0.0007797902682796121, "clip_ratio/high_mean": 0.0007797902682796121, "clip_ratio/low_mean": 4.9364475125912576e-05, "clip_ratio/low_min": 4.9364475125912576e-05, "clip_ratio/region_mean": 0.0008291547477710992, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 838.8800048828125, "completions/mean_terminated_length": 838.8800048828125, "completions/min_length": 474.0, "completions/min_terminated_length": 474.0, "entropy": 0.17070605158805846, "epoch": 0.14130434782608695, "frac_reward_zero_std": 0.0, "grad_norm": 0.015239793807268143, "learning_rate": 5e-05, "loss": 0.03420329466462135, "num_tokens": 3904178.0, "reward": 0.3903906047344208, "reward_std": 0.41191530227661133, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.4096093773841858, "rewards/length_penalty/std": 0.11314770579338074, "sampling/importance_sampling_ratio/max": 2.1938090324401855, "sampling/importance_sampling_ratio/mean": 0.9938638210296631, "sampling/importance_sampling_ratio/min": 0.08321992307901382, "sampling/sampling_logp_difference/max": 2.4862685203552246, "sampling/sampling_logp_difference/mean": 0.01698494143784046, "step": 52, "step_time": 16.543803959153593 }, { "clip_ratio/high_max": 0.0008488356252200902, "clip_ratio/high_mean": 0.0008488356252200902, "clip_ratio/low_mean": 4.159114178037271e-05, "clip_ratio/low_min": 4.159114178037271e-05, "clip_ratio/region_mean": 0.0008904267684556544, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 1766.0, "completions/mean_length": 1033.179931640625, "completions/mean_terminated_length": 944.934814453125, "completions/min_length": 453.0, "completions/min_terminated_length": 453.0, "entropy": 0.20474017262458802, "epoch": 0.14402173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.0204797200858593, "learning_rate": 5e-05, "loss": 0.05927962809801102, "num_tokens": 3958557.0, "reward": 0.19551756978034973, "reward_std": 0.6273199915885925, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.5044824481010437, "rewards/length_penalty/std": 0.22523540258407593, "sampling/importance_sampling_ratio/max": 1.9548008441925049, "sampling/importance_sampling_ratio/mean": 0.991944432258606, "sampling/importance_sampling_ratio/min": 0.04066736251115799, "sampling/sampling_logp_difference/max": 3.202329397201538, "sampling/sampling_logp_difference/mean": 0.0187305249273777, "step": 53, "step_time": 23.953327698865905 }, { "clip_ratio/high_max": 0.0005767365772044286, "clip_ratio/high_mean": 0.0005767365772044286, "clip_ratio/low_mean": 6.257046188693494e-05, "clip_ratio/low_min": 6.257046188693494e-05, "clip_ratio/region_mean": 0.0006393070274498313, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1871.0, "completions/mean_length": 1254.1199951171875, "completions/mean_terminated_length": 1055.6500244140625, "completions/min_length": 588.0, "completions/min_terminated_length": 588.0, "entropy": 0.20505343973636628, "epoch": 0.14673913043478262, "frac_reward_zero_std": 0.0, "grad_norm": 0.018724745139479637, "learning_rate": 5e-05, "loss": 0.07014630734920502, "num_tokens": 4025083.0, "reward": 0.14763671159744263, "reward_std": 0.6248001456260681, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.6123632788658142, "rewards/length_penalty/std": 0.24844162166118622, "sampling/importance_sampling_ratio/max": 2.6337807178497314, "sampling/importance_sampling_ratio/mean": 0.992132842540741, "sampling/importance_sampling_ratio/min": 0.11581636965274811, "sampling/sampling_logp_difference/max": 2.1557493209838867, "sampling/sampling_logp_difference/mean": 0.01712055131793022, "step": 54, "step_time": 25.70337758003734 }, { "clip_ratio/high_max": 0.0005223329411819577, "clip_ratio/high_mean": 0.0005223329411819577, "clip_ratio/low_mean": 0.00012916228006361053, "clip_ratio/low_min": 0.00012916228006361053, "clip_ratio/region_mean": 0.0006514952285215259, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 1897.0, "completions/mean_length": 1253.8399658203125, "completions/mean_terminated_length": 913.4857177734375, "completions/min_length": 545.0, "completions/min_terminated_length": 545.0, "entropy": 0.24083685576915742, "epoch": 0.14945652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.02363269031047821, "learning_rate": 5e-05, "loss": 0.03444070741534233, "num_tokens": 4090665.0, "reward": 0.08777343481779099, "reward_std": 0.7344273328781128, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.6122265458106995, "rewards/length_penalty/std": 0.303725928068161, "sampling/importance_sampling_ratio/max": 2.8206093311309814, "sampling/importance_sampling_ratio/mean": 0.9912818670272827, "sampling/importance_sampling_ratio/min": 0.14987419545650482, "sampling/sampling_logp_difference/max": 1.8979589939117432, "sampling/sampling_logp_difference/mean": 0.019693711772561073, "step": 55, "step_time": 25.123637669952586 }, { "clip_ratio/high_max": 0.0012042871094308794, "clip_ratio/high_mean": 0.0012042871094308794, "clip_ratio/low_mean": 0.00010765771730802953, "clip_ratio/low_min": 0.00010765771730802953, "clip_ratio/region_mean": 0.0013119448209181427, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1432.0, "completions/mean_length": 987.5599975585938, "completions/mean_terminated_length": 722.4500122070312, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "entropy": 0.1985310882329941, "epoch": 0.15217391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.014590896666049957, "learning_rate": 5e-05, "loss": 0.029594430699944496, "num_tokens": 4143113.0, "reward": 0.11779296398162842, "reward_std": 0.673450767993927, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.4822070300579071, "rewards/length_penalty/std": 0.29239600896835327, "sampling/importance_sampling_ratio/max": 2.725640296936035, "sampling/importance_sampling_ratio/mean": 0.9925079345703125, "sampling/importance_sampling_ratio/min": 0.15510861575603485, "sampling/sampling_logp_difference/max": 1.863629698753357, "sampling/sampling_logp_difference/mean": 0.017994750291109085, "step": 56, "step_time": 24.02179194591008 }, { "clip_ratio/high_max": 0.0005607875806163065, "clip_ratio/high_mean": 0.0005607875806163065, "clip_ratio/low_mean": 0.0001157178936409764, "clip_ratio/low_min": 0.0001157178936409764, "clip_ratio/region_mean": 0.0006765054655261338, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1959.0, "completions/mean_length": 1183.419921875, "completions/mean_terminated_length": 967.2750244140625, "completions/min_length": 448.0, "completions/min_terminated_length": 448.0, "entropy": 0.24734485745429993, "epoch": 0.15489130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.01673017628490925, "learning_rate": 5e-05, "loss": 0.04797259345650673, "num_tokens": 4205524.0, "reward": 0.22215819358825684, "reward_std": 0.6535060405731201, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5778418183326721, "rewards/length_penalty/std": 0.30248209834098816, "sampling/importance_sampling_ratio/max": 2.1797726154327393, "sampling/importance_sampling_ratio/mean": 0.9906326532363892, "sampling/importance_sampling_ratio/min": 0.17795737087726593, "sampling/sampling_logp_difference/max": 1.7262113094329834, "sampling/sampling_logp_difference/mean": 0.019809162244200706, "step": 57, "step_time": 25.37448728713207 }, { "clip_ratio/high_max": 0.0006756895920261741, "clip_ratio/high_mean": 0.0006756895920261741, "clip_ratio/low_mean": 0.0001676665517152287, "clip_ratio/low_min": 0.0001676665517152287, "clip_ratio/region_mean": 0.0008433561422862112, "completions/clipped_ratio": 0.23999999463558197, "completions/max_length": 2048.0, "completions/max_terminated_length": 1995.0, "completions/mean_length": 1314.719970703125, "completions/mean_terminated_length": 1083.157958984375, "completions/min_length": 486.0, "completions/min_terminated_length": 486.0, "entropy": 0.24679793417453766, "epoch": 0.15760869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.023087836802005768, "learning_rate": 5e-05, "loss": 0.07203301787376404, "num_tokens": 4275070.0, "reward": -0.001953125, "reward_std": 0.7374632358551025, "rewards/correctness/mean": 0.6399999856948853, "rewards/correctness/std": 0.4848732054233551, "rewards/length_penalty/mean": -0.6419531106948853, "rewards/length_penalty/std": 0.2896905243396759, "sampling/importance_sampling_ratio/max": 2.4132602214813232, "sampling/importance_sampling_ratio/mean": 0.9908359050750732, "sampling/importance_sampling_ratio/min": 0.09266611188650131, "sampling/sampling_logp_difference/max": 2.3787524700164795, "sampling/sampling_logp_difference/mean": 0.020349619910120964, "step": 58, "step_time": 25.843243841081858 }, { "clip_ratio/high_max": 0.0007752020144835114, "clip_ratio/high_mean": 0.0007752020144835114, "clip_ratio/low_mean": 0.0001578759402036667, "clip_ratio/low_min": 0.0001578759402036667, "clip_ratio/region_mean": 0.000933077948866412, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 1734.0, "completions/mean_length": 909.1599731445312, "completions/mean_terminated_length": 810.1304321289062, "completions/min_length": 438.0, "completions/min_terminated_length": 438.0, "entropy": 0.1871820569038391, "epoch": 0.16032608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.021188728511333466, "learning_rate": 5e-05, "loss": 0.07215945422649384, "num_tokens": 4323238.0, "reward": 0.4360741972923279, "reward_std": 0.5008804798126221, "rewards/correctness/mean": 0.8799999952316284, "rewards/correctness/std": 0.32826071977615356, "rewards/length_penalty/mean": -0.44392576813697815, "rewards/length_penalty/std": 0.22381433844566345, "sampling/importance_sampling_ratio/max": 2.9011504650115967, "sampling/importance_sampling_ratio/mean": 0.9930024743080139, "sampling/importance_sampling_ratio/min": 0.1250293254852295, "sampling/sampling_logp_difference/max": 2.079206943511963, "sampling/sampling_logp_difference/mean": 0.018969286233186722, "step": 59, "step_time": 23.926262149121612 }, { "clip_ratio/high_max": 0.0004012422810774297, "clip_ratio/high_mean": 0.0004012422810774297, "clip_ratio/low_mean": 4.353504627943039e-05, "clip_ratio/low_min": 4.353504627943039e-05, "clip_ratio/region_mean": 0.00044477732735686003, "completions/clipped_ratio": 0.0, "completions/max_length": 1467.0, "completions/max_terminated_length": 1467.0, "completions/mean_length": 893.9400024414062, "completions/mean_terminated_length": 893.9400024414062, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.15323590338230134, "epoch": 0.16304347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.013462953269481659, "learning_rate": 5e-05, "loss": 0.026647938415408134, "num_tokens": 4370745.0, "reward": -0.05649413913488388, "reward_std": 0.5149266123771667, "rewards/correctness/mean": 0.3799999952316284, "rewards/correctness/std": 0.4903143644332886, "rewards/length_penalty/mean": -0.4364941418170929, "rewards/length_penalty/std": 0.13239894807338715, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9946074485778809, "sampling/importance_sampling_ratio/min": 0.07585878670215607, "sampling/sampling_logp_difference/max": 2.5788817405700684, "sampling/sampling_logp_difference/mean": 0.015621195547282696, "step": 60, "step_time": 17.78750504530035 }, { "clip_ratio/high_max": 0.0007783836219459773, "clip_ratio/high_mean": 0.0007783836219459773, "clip_ratio/low_mean": 7.167461153585463e-05, "clip_ratio/low_min": 7.167461153585463e-05, "clip_ratio/region_mean": 0.0008500582538545131, "completions/clipped_ratio": 0.0, "completions/max_length": 1332.0, "completions/max_terminated_length": 1332.0, "completions/mean_length": 816.8800048828125, "completions/mean_terminated_length": 816.8800048828125, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.16096674501895905, "epoch": 0.16576086956521738, "frac_reward_zero_std": 0.0, "grad_norm": 0.01674392633140087, "learning_rate": 5e-05, "loss": 0.04878897964954376, "num_tokens": 4413999.0, "reward": 0.6011328101158142, "reward_std": 0.08173488825559616, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.3988671898841858, "rewards/length_penalty/std": 0.08173488825559616, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9938045740127563, "sampling/importance_sampling_ratio/min": 0.03371766582131386, "sampling/sampling_logp_difference/max": 3.38973331451416, "sampling/sampling_logp_difference/mean": 0.01792747713625431, "step": 61, "step_time": 15.884311718866229 }, { "clip_ratio/high_max": 0.0005490639596246183, "clip_ratio/high_mean": 0.0005490639596246183, "clip_ratio/low_mean": 8.991758077172562e-05, "clip_ratio/low_min": 8.991758077172562e-05, "clip_ratio/region_mean": 0.0006389815418515355, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 1749.0, "completions/mean_length": 1040.5399169921875, "completions/mean_terminated_length": 976.2340087890625, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.15563784837722777, "epoch": 0.16847826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.018042365089058876, "learning_rate": 5e-05, "loss": 0.07988356798887253, "num_tokens": 4469646.0, "reward": 0.33192381262779236, "reward_std": 0.5288719534873962, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.5080761909484863, "rewards/length_penalty/std": 0.21403193473815918, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9942488074302673, "sampling/importance_sampling_ratio/min": 0.06006976589560509, "sampling/sampling_logp_difference/max": 2.812248706817627, "sampling/sampling_logp_difference/mean": 0.015519483014941216, "step": 62, "step_time": 24.003339575137943 }, { "clip_ratio/high_max": 0.0007015687064267695, "clip_ratio/high_mean": 0.0007015687064267695, "clip_ratio/low_mean": 0.00010667735186871141, "clip_ratio/low_min": 0.00010667735186871141, "clip_ratio/region_mean": 0.0008082460495643318, "completions/clipped_ratio": 0.0, "completions/max_length": 1687.0, "completions/max_terminated_length": 1687.0, "completions/mean_length": 828.7799682617188, "completions/mean_terminated_length": 828.7799682617188, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.16558827161788942, "epoch": 0.17119565217391305, "frac_reward_zero_std": 0.0, "grad_norm": 0.016367904841899872, "learning_rate": 5e-05, "loss": 0.04421332851052284, "num_tokens": 4514335.0, "reward": 0.3953222632408142, "reward_std": 0.5528039932250977, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.40467774868011475, "rewards/length_penalty/std": 0.1717558354139328, "sampling/importance_sampling_ratio/max": 2.7936859130859375, "sampling/importance_sampling_ratio/mean": 0.9938439130783081, "sampling/importance_sampling_ratio/min": 0.05281807854771614, "sampling/sampling_logp_difference/max": 2.940901756286621, "sampling/sampling_logp_difference/mean": 0.017519377171993256, "step": 63, "step_time": 20.010241974843666 }, { "clip_ratio/high_max": 0.0006772913970053196, "clip_ratio/high_mean": 0.0006772913970053196, "clip_ratio/low_mean": 6.347343005472795e-05, "clip_ratio/low_min": 6.347343005472795e-05, "clip_ratio/region_mean": 0.0007407648256048561, "completions/clipped_ratio": 0.0, "completions/max_length": 2002.0, "completions/max_terminated_length": 2002.0, "completions/mean_length": 881.5399780273438, "completions/mean_terminated_length": 881.5399780273438, "completions/min_length": 503.0, "completions/min_terminated_length": 503.0, "entropy": 0.16818468570709227, "epoch": 0.17391304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.016808470711112022, "learning_rate": 5e-05, "loss": 0.010990206152200699, "num_tokens": 4561932.0, "reward": 0.5095605254173279, "reward_std": 0.27923068404197693, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979365825653, "rewards/length_penalty/mean": -0.43043944239616394, "rewards/length_penalty/std": 0.12210877984762192, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9936347007751465, "sampling/importance_sampling_ratio/min": 0.11113839596509933, "sampling/sampling_logp_difference/max": 2.19697904586792, "sampling/sampling_logp_difference/mean": 0.018134895712137222, "step": 64, "step_time": 22.430961188860238 }, { "clip_ratio/high_max": 0.0006107009830884635, "clip_ratio/high_mean": 0.0006107009830884635, "clip_ratio/low_mean": 0.00012073849065927789, "clip_ratio/low_min": 0.00012073849065927789, "clip_ratio/region_mean": 0.000731439475202933, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 1452.0, "completions/mean_length": 999.5, "completions/mean_terminated_length": 828.81396484375, "completions/min_length": 454.0, "completions/min_terminated_length": 454.0, "entropy": 0.2120996743440628, "epoch": 0.1766304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.01858065277338028, "learning_rate": 5e-05, "loss": 0.07004983723163605, "num_tokens": 4615617.0, "reward": 0.3319628834724426, "reward_std": 0.5958665013313293, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.488037109375, "rewards/length_penalty/std": 0.233457550406456, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9916940331459045, "sampling/importance_sampling_ratio/min": 0.0842479020357132, "sampling/sampling_logp_difference/max": 2.473991632461548, "sampling/sampling_logp_difference/mean": 0.020551443099975586, "step": 65, "step_time": 23.88459292682819 }, { "clip_ratio/high_max": 0.0006235995824681595, "clip_ratio/high_mean": 0.0006235995824681595, "clip_ratio/low_mean": 0.00022068115940783174, "clip_ratio/low_min": 0.00022068115940783174, "clip_ratio/region_mean": 0.0008442807185929269, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 1759.0, "completions/mean_length": 875.4400024414062, "completions/mean_terminated_length": 684.5581665039062, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.2801995426416397, "epoch": 0.1793478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.01836577244102955, "learning_rate": 5e-05, "loss": 0.0533025860786438, "num_tokens": 4662439.0, "reward": 0.37253904342651367, "reward_std": 0.6679306626319885, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.4274609386920929, "rewards/length_penalty/std": 0.2749612033367157, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9891699552536011, "sampling/importance_sampling_ratio/min": 0.09494689106941223, "sampling/sampling_logp_difference/max": 2.3544375896453857, "sampling/sampling_logp_difference/mean": 0.025045860558748245, "step": 66, "step_time": 24.01824735174887 }, { "clip_ratio/high_max": 0.0006467935570981353, "clip_ratio/high_mean": 0.0006467935570981353, "clip_ratio/low_mean": 4.287245392333716e-05, "clip_ratio/low_min": 4.287245392333716e-05, "clip_ratio/region_mean": 0.0006896660139318556, "completions/clipped_ratio": 0.0, "completions/max_length": 1456.0, "completions/max_terminated_length": 1456.0, "completions/mean_length": 827.5399780273438, "completions/mean_terminated_length": 827.5399780273438, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.1334616631269455, "epoch": 0.18206521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.015385694801807404, "learning_rate": 5e-05, "loss": 0.042910344898700714, "num_tokens": 4706766.0, "reward": 0.1959277242422104, "reward_std": 0.5535297393798828, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.40407225489616394, "rewards/length_penalty/std": 0.1448303610086441, "sampling/importance_sampling_ratio/max": 2.695427417755127, "sampling/importance_sampling_ratio/mean": 0.9948953986167908, "sampling/importance_sampling_ratio/min": 0.09551125019788742, "sampling/sampling_logp_difference/max": 2.348511219024658, "sampling/sampling_logp_difference/mean": 0.01522344071418047, "step": 67, "step_time": 16.992219537962228 }, { "clip_ratio/high_max": 0.0005427057622000575, "clip_ratio/high_mean": 0.0005427057622000575, "clip_ratio/low_mean": 6.446797924581915e-05, "clip_ratio/low_min": 6.446797924581915e-05, "clip_ratio/region_mean": 0.0006071737268939614, "completions/clipped_ratio": 0.0, "completions/max_length": 1989.0, "completions/max_terminated_length": 1989.0, "completions/mean_length": 871.8800048828125, "completions/mean_terminated_length": 871.8800048828125, "completions/min_length": 508.0, "completions/min_terminated_length": 508.0, "entropy": 0.1475384384393692, "epoch": 0.18478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.017730075865983963, "learning_rate": 5e-05, "loss": 0.05759969353675842, "num_tokens": 4754120.0, "reward": 0.3742773234844208, "reward_std": 0.5240140557289124, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.4257226586341858, "rewards/length_penalty/std": 0.14220459759235382, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9945562481880188, "sampling/importance_sampling_ratio/min": 0.09534671157598495, "sampling/sampling_logp_difference/max": 2.3502354621887207, "sampling/sampling_logp_difference/mean": 0.01613455079495907, "step": 68, "step_time": 22.368666774127632 }, { "clip_ratio/high_max": 0.0006822091600042768, "clip_ratio/high_mean": 0.0006822091600042768, "clip_ratio/low_mean": 0.00017659609293332323, "clip_ratio/low_min": 0.00017659609293332323, "clip_ratio/region_mean": 0.0008588052645791322, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 1823.0, "completions/mean_length": 852.5, "completions/mean_terminated_length": 624.7857055664062, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "entropy": 0.22420361936092376, "epoch": 0.1875, "frac_reward_zero_std": 0.0, "grad_norm": 0.01747017167508602, "learning_rate": 5e-05, "loss": 0.045437250286340714, "num_tokens": 4799125.0, "reward": 0.18374022841453552, "reward_std": 0.6997568011283875, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.416259765625, "rewards/length_penalty/std": 0.2845315635204315, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.99150550365448, "sampling/importance_sampling_ratio/min": 0.08363517373800278, "sampling/sampling_logp_difference/max": 2.4812910556793213, "sampling/sampling_logp_difference/mean": 0.02138812094926834, "step": 69, "step_time": 23.31697426387109 }, { "clip_ratio/high_max": 0.0008050206408370287, "clip_ratio/high_mean": 0.0008050206408370287, "clip_ratio/low_mean": 5.3399032913148405e-05, "clip_ratio/low_min": 5.3399032913148405e-05, "clip_ratio/region_mean": 0.000858419673750177, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1751.0, "completions/mean_length": 1082.97998046875, "completions/mean_terminated_length": 841.7250366210938, "completions/min_length": 435.0, "completions/min_terminated_length": 435.0, "entropy": 0.14042216837406157, "epoch": 0.19021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.01626087725162506, "learning_rate": 5e-05, "loss": 0.04337764531373978, "num_tokens": 4856784.0, "reward": -0.3287988305091858, "reward_std": 0.583001434803009, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5287988185882568, "rewards/length_penalty/std": 0.2958765923976898, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9947971701622009, "sampling/importance_sampling_ratio/min": 0.1584167629480362, "sampling/sampling_logp_difference/max": 1.8425259590148926, "sampling/sampling_logp_difference/mean": 0.014615754596889019, "step": 70, "step_time": 24.817883169045672 }, { "clip_ratio/high_max": 0.0006307939882390201, "clip_ratio/high_mean": 0.0006307939882390201, "clip_ratio/low_mean": 0.00010670171468518674, "clip_ratio/low_min": 0.00010670171468518674, "clip_ratio/region_mean": 0.0007374957029242068, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1123.0, "completions/mean_length": 677.9400024414062, "completions/mean_terminated_length": 620.8541870117188, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.17736737430095673, "epoch": 0.19293478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.015899593010544777, "learning_rate": 5e-05, "loss": 0.04084380343556404, "num_tokens": 4893041.0, "reward": 0.20897459983825684, "reward_std": 0.5713562965393066, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.3310253918170929, "rewards/length_penalty/std": 0.16477957367897034, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9929877519607544, "sampling/importance_sampling_ratio/min": 0.0540241040289402, "sampling/sampling_logp_difference/max": 2.9183249473571777, "sampling/sampling_logp_difference/mean": 0.02107720635831356, "step": 71, "step_time": 22.17649055412039 }, { "clip_ratio/high_max": 0.0007707292737904936, "clip_ratio/high_mean": 0.0007707292737904936, "clip_ratio/low_mean": 9.523197077214718e-05, "clip_ratio/low_min": 9.523197077214718e-05, "clip_ratio/region_mean": 0.0008659612271003425, "completions/clipped_ratio": 0.0, "completions/max_length": 1150.0, "completions/max_terminated_length": 1150.0, "completions/mean_length": 638.5599975585938, "completions/mean_terminated_length": 638.5599975585938, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.16127082109451293, "epoch": 0.1956521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.015706723555922508, "learning_rate": 5e-05, "loss": 0.012553978711366653, "num_tokens": 4928639.0, "reward": 0.42820310592651367, "reward_std": 0.5053699612617493, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.3117968738079071, "rewards/length_penalty/std": 0.0972241684794426, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9939435124397278, "sampling/importance_sampling_ratio/min": 0.07872314006090164, "sampling/sampling_logp_difference/max": 2.541818141937256, "sampling/sampling_logp_difference/mean": 0.020169558003544807, "step": 72, "step_time": 13.558618172770366 }, { "clip_ratio/high_max": 0.0005829828616697342, "clip_ratio/high_mean": 0.0005829828616697342, "clip_ratio/low_mean": 0.00014270264655351638, "clip_ratio/low_min": 0.00014270264655351638, "clip_ratio/region_mean": 0.0007256855198647827, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 1980.0, "completions/mean_length": 888.760009765625, "completions/mean_terminated_length": 787.95654296875, "completions/min_length": 440.0, "completions/min_terminated_length": 440.0, "entropy": 0.19430480599403382, "epoch": 0.1983695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.023028841242194176, "learning_rate": 5e-05, "loss": 0.03241705894470215, "num_tokens": 4975487.0, "reward": 0.22603514790534973, "reward_std": 0.6428487300872803, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.47851812839508057, "rewards/length_penalty/mean": -0.4339648485183716, "rewards/length_penalty/std": 0.2522260248661041, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.992754340171814, "sampling/importance_sampling_ratio/min": 0.1667802333831787, "sampling/sampling_logp_difference/max": 1.7910783290863037, "sampling/sampling_logp_difference/mean": 0.019066043198108673, "step": 73, "step_time": 23.85767387226224 }, { "clip_ratio/high_max": 0.0005934039130806923, "clip_ratio/high_mean": 0.0005934039130806923, "clip_ratio/low_mean": 0.00011007156572304667, "clip_ratio/low_min": 0.00011007156572304667, "clip_ratio/region_mean": 0.0007034754729829728, "completions/clipped_ratio": 0.0, "completions/max_length": 1745.0, "completions/max_terminated_length": 1745.0, "completions/mean_length": 641.0799560546875, "completions/mean_terminated_length": 641.0799560546875, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.1928534835577011, "epoch": 0.20108695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.019313158467411995, "learning_rate": 5e-05, "loss": 0.05762927606701851, "num_tokens": 5010871.0, "reward": 0.08697265386581421, "reward_std": 0.5515088438987732, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.31302735209465027, "rewards/length_penalty/std": 0.12540066242218018, "sampling/importance_sampling_ratio/max": 2.433044910430908, "sampling/importance_sampling_ratio/mean": 0.9928313493728638, "sampling/importance_sampling_ratio/min": 0.08987344801425934, "sampling/sampling_logp_difference/max": 2.4093527793884277, "sampling/sampling_logp_difference/mean": 0.021775977686047554, "step": 74, "step_time": 19.006243550917134 }, { "clip_ratio/high_max": 0.000405387906357646, "clip_ratio/high_mean": 0.000405387906357646, "clip_ratio/low_mean": 0.00012361486005829647, "clip_ratio/low_min": 0.00012361486005829647, "clip_ratio/region_mean": 0.0005290027736919, "completions/clipped_ratio": 0.0, "completions/max_length": 1529.0, "completions/max_terminated_length": 1529.0, "completions/mean_length": 633.239990234375, "completions/mean_terminated_length": 633.239990234375, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.17757215201854706, "epoch": 0.20380434782608695, "frac_reward_zero_std": 0.0, "grad_norm": 0.020598672330379486, "learning_rate": 5e-05, "loss": 0.03665365278720856, "num_tokens": 5044863.0, "reward": 0.49080076813697815, "reward_std": 0.45810505747795105, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.3091992139816284, "rewards/length_penalty/std": 0.1596784144639969, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9929429292678833, "sampling/importance_sampling_ratio/min": 0.06892305612564087, "sampling/sampling_logp_difference/max": 2.674764633178711, "sampling/sampling_logp_difference/mean": 0.021487612277269363, "step": 75, "step_time": 16.96747936704196 }, { "clip_ratio/high_max": 0.0008531624858733267, "clip_ratio/high_mean": 0.0008531624858733267, "clip_ratio/low_mean": 1.858218020061031e-05, "clip_ratio/low_min": 1.858218020061031e-05, "clip_ratio/region_mean": 0.0008717446587979793, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1162.0, "completions/mean_length": 658.0599975585938, "completions/mean_terminated_length": 503.6222229003906, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "entropy": 0.24603551626205444, "epoch": 0.20652173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.012405374087393284, "learning_rate": 5e-05, "loss": 0.025707000866532326, "num_tokens": 5081106.0, "reward": 0.43868163228034973, "reward_std": 0.6422238349914551, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.3213183581829071, "rewards/length_penalty/std": 0.255402535200119, "sampling/importance_sampling_ratio/max": 2.560499906539917, "sampling/importance_sampling_ratio/mean": 0.9905421733856201, "sampling/importance_sampling_ratio/min": 0.1971578299999237, "sampling/sampling_logp_difference/max": 1.6237506866455078, "sampling/sampling_logp_difference/mean": 0.0244895052164793, "step": 76, "step_time": 22.57146995374933 }, { "clip_ratio/high_max": 0.0003801504441071302, "clip_ratio/high_mean": 0.0003801504441071302, "clip_ratio/low_mean": 0.00011223704786971212, "clip_ratio/low_min": 0.00011223704786971212, "clip_ratio/region_mean": 0.0004923874977976084, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1808.0, "completions/mean_length": 628.7799682617188, "completions/mean_terminated_length": 569.6458740234375, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.20442797541618346, "epoch": 0.20923913043478262, "frac_reward_zero_std": 0.0, "grad_norm": 0.019539417698979378, "learning_rate": 5e-05, "loss": 0.04475327581167221, "num_tokens": 5115105.0, "reward": 0.552978515625, "reward_std": 0.547501802444458, "rewards/correctness/mean": 0.8600000143051147, "rewards/correctness/std": 0.3505098223686218, "rewards/length_penalty/mean": -0.30702149868011475, "rewards/length_penalty/std": 0.22665424644947052, "sampling/importance_sampling_ratio/max": 2.7733845710754395, "sampling/importance_sampling_ratio/mean": 0.992488443851471, "sampling/importance_sampling_ratio/min": 0.21041618287563324, "sampling/sampling_logp_difference/max": 1.5586678981781006, "sampling/sampling_logp_difference/mean": 0.021170448511838913, "step": 77, "step_time": 22.34465615078807 }, { "clip_ratio/high_max": 0.0006306407682131976, "clip_ratio/high_mean": 0.0006306407682131976, "clip_ratio/low_mean": 3.474635304883123e-05, "clip_ratio/low_min": 3.474635304883123e-05, "clip_ratio/region_mean": 0.0006653871096204966, "completions/clipped_ratio": 0.0, "completions/max_length": 1658.0, "completions/max_terminated_length": 1658.0, "completions/mean_length": 599.6599731445312, "completions/mean_terminated_length": 599.6599731445312, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.1244092807173729, "epoch": 0.21195652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.015446798875927925, "learning_rate": 5e-05, "loss": 0.033272288739681244, "num_tokens": 5147388.0, "reward": 0.26719725131988525, "reward_std": 0.5827784538269043, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.29280272126197815, "rewards/length_penalty/std": 0.13731949031352997, "sampling/importance_sampling_ratio/max": 2.4936909675598145, "sampling/importance_sampling_ratio/mean": 0.994698166847229, "sampling/importance_sampling_ratio/min": 0.31123581528663635, "sampling/sampling_logp_difference/max": 1.1672043800354004, "sampling/sampling_logp_difference/mean": 0.0171463992446661, "step": 78, "step_time": 18.60519856400788 }, { "clip_ratio/high_max": 0.0008912550430977717, "clip_ratio/high_mean": 0.0008912550430977717, "clip_ratio/low_mean": 6.107631779741495e-05, "clip_ratio/low_min": 6.107631779741495e-05, "clip_ratio/region_mean": 0.0009523313667159528, "completions/clipped_ratio": 0.0, "completions/max_length": 1229.0, "completions/max_terminated_length": 1229.0, "completions/mean_length": 649.219970703125, "completions/mean_terminated_length": 649.219970703125, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.13126298934221267, "epoch": 0.21467391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.0188016165047884, "learning_rate": 5e-05, "loss": 0.015198985114693642, "num_tokens": 5183309.0, "reward": 0.4429980516433716, "reward_std": 0.4340318739414215, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.31700193881988525, "rewards/length_penalty/std": 0.08376923203468323, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9953542351722717, "sampling/importance_sampling_ratio/min": 0.24312418699264526, "sampling/sampling_logp_difference/max": 1.5119855403900146, "sampling/sampling_logp_difference/mean": 0.018735092133283615, "step": 79, "step_time": 14.177364142145962 }, { "clip_ratio/high_max": 0.0004742158984299749, "clip_ratio/high_mean": 0.0004742158984299749, "clip_ratio/low_mean": 0.0001656829408602789, "clip_ratio/low_min": 0.0001656829408602789, "clip_ratio/region_mean": 0.0006398988363798708, "completions/clipped_ratio": 0.0, "completions/max_length": 1211.0, "completions/max_terminated_length": 1211.0, "completions/mean_length": 603.4199829101562, "completions/mean_terminated_length": 603.4199829101562, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.15212061405181884, "epoch": 0.21739130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.0165855772793293, "learning_rate": 5e-05, "loss": 0.041541777551174164, "num_tokens": 5216640.0, "reward": 0.10536132752895355, "reward_std": 0.5482556223869324, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.29463866353034973, "rewards/length_penalty/std": 0.07611750066280365, "sampling/importance_sampling_ratio/max": 2.329467535018921, "sampling/importance_sampling_ratio/mean": 0.9940717816352844, "sampling/importance_sampling_ratio/min": 0.20993992686271667, "sampling/sampling_logp_difference/max": 1.5609338283538818, "sampling/sampling_logp_difference/mean": 0.019620422273874283, "step": 80, "step_time": 13.800314102787524 }, { "clip_ratio/high_max": 0.000613589191925712, "clip_ratio/high_mean": 0.000613589191925712, "clip_ratio/low_mean": 8.642562024760992e-05, "clip_ratio/low_min": 8.642562024760992e-05, "clip_ratio/region_mean": 0.0007000148179940879, "completions/clipped_ratio": 0.0, "completions/max_length": 1088.0, "completions/max_terminated_length": 1088.0, "completions/mean_length": 649.4199829101562, "completions/mean_terminated_length": 649.4199829101562, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.15520131587982178, "epoch": 0.22010869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.020023388788104057, "learning_rate": 5e-05, "loss": 0.010980712249875069, "num_tokens": 5253571.0, "reward": 0.42290037870407104, "reward_std": 0.3821259140968323, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308748841285706, "rewards/length_penalty/mean": -0.31709960103034973, "rewards/length_penalty/std": 0.09378077834844589, "sampling/importance_sampling_ratio/max": 2.707496404647827, "sampling/importance_sampling_ratio/mean": 0.9943109154701233, "sampling/importance_sampling_ratio/min": 0.11866816133260727, "sampling/sampling_logp_difference/max": 2.1314241886138916, "sampling/sampling_logp_difference/mean": 0.0196909848600626, "step": 81, "step_time": 13.374773803167045 }, { "clip_ratio/high_max": 0.0003143954527331516, "clip_ratio/high_mean": 0.0003143954527331516, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0003143954527331516, "completions/clipped_ratio": 0.0, "completions/max_length": 748.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 444.5799865722656, "completions/mean_terminated_length": 444.5799865722656, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.1351422816514969, "epoch": 0.22282608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.013173939660191536, "learning_rate": 5e-05, "loss": 0.020658979192376137, "num_tokens": 5278650.0, "reward": 0.5829198956489563, "reward_std": 0.46361181139945984, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.21708007156848907, "rewards/length_penalty/std": 0.06732401251792908, "sampling/importance_sampling_ratio/max": 2.7757651805877686, "sampling/importance_sampling_ratio/mean": 0.9944673776626587, "sampling/importance_sampling_ratio/min": 0.31246867775917053, "sampling/sampling_logp_difference/max": 1.1632510423660278, "sampling/sampling_logp_difference/mean": 0.021512571722269058, "step": 82, "step_time": 9.464399790856987 }, { "clip_ratio/high_max": 0.0006479248695541173, "clip_ratio/high_mean": 0.0006479248695541173, "clip_ratio/low_mean": 0.00010048146359622478, "clip_ratio/low_min": 0.00010048146359622478, "clip_ratio/region_mean": 0.0007484063331503421, "completions/clipped_ratio": 0.0, "completions/max_length": 720.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 408.0799865722656, "completions/mean_terminated_length": 408.0799865722656, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.1544148862361908, "epoch": 0.22554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.014162655919790268, "learning_rate": 5e-05, "loss": 0.009620768949389458, "num_tokens": 5301484.0, "reward": 0.3607421815395355, "reward_std": 0.4975096583366394, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.19925780594348907, "rewards/length_penalty/std": 0.06029851362109184, "sampling/importance_sampling_ratio/max": 2.475653648376465, "sampling/importance_sampling_ratio/mean": 0.9939197301864624, "sampling/importance_sampling_ratio/min": 0.22558780014514923, "sampling/sampling_logp_difference/max": 1.4890458583831787, "sampling/sampling_logp_difference/mean": 0.023020122200250626, "step": 83, "step_time": 8.50258247810416 }, { "clip_ratio/high_max": 0.0009030202636495233, "clip_ratio/high_mean": 0.0009030202636495233, "clip_ratio/low_mean": 7.876795716583729e-05, "clip_ratio/low_min": 7.876795716583729e-05, "clip_ratio/region_mean": 0.0009817882324568926, "completions/clipped_ratio": 0.0, "completions/max_length": 1560.0, "completions/max_terminated_length": 1560.0, "completions/mean_length": 530.8399658203125, "completions/mean_terminated_length": 530.8399658203125, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.14245042204856873, "epoch": 0.22826086956521738, "frac_reward_zero_std": 0.0, "grad_norm": 0.014098316431045532, "learning_rate": 5e-05, "loss": 0.028290294110774994, "num_tokens": 5331006.0, "reward": 0.5408007502555847, "reward_std": 0.3885652720928192, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.25919923186302185, "rewards/length_penalty/std": 0.1674625128507614, "sampling/importance_sampling_ratio/max": 2.278921604156494, "sampling/importance_sampling_ratio/mean": 0.9943795800209045, "sampling/importance_sampling_ratio/min": 0.21966639161109924, "sampling/sampling_logp_difference/max": 1.5156453847885132, "sampling/sampling_logp_difference/mean": 0.019804934039711952, "step": 84, "step_time": 17.037076706066728 }, { "clip_ratio/high_max": 0.0018277494236826897, "clip_ratio/high_mean": 0.0018277494236826897, "clip_ratio/low_mean": 0.00013214687642175703, "clip_ratio/low_min": 0.00013214687642175703, "clip_ratio/region_mean": 0.001959896294283681, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 2027.0, "completions/mean_length": 700.8399658203125, "completions/mean_terminated_length": 644.7083740234375, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.2856750160455704, "epoch": 0.23097826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.026974737644195557, "learning_rate": 5e-05, "loss": 0.09304343163967133, "num_tokens": 5368068.0, "reward": 0.3377929627895355, "reward_std": 0.5755784511566162, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.34220704436302185, "rewards/length_penalty/std": 0.2737504541873932, "sampling/importance_sampling_ratio/max": 2.973176956176758, "sampling/importance_sampling_ratio/mean": 0.988871157169342, "sampling/importance_sampling_ratio/min": 0.13172246515750885, "sampling/sampling_logp_difference/max": 2.0270581245422363, "sampling/sampling_logp_difference/mean": 0.028557635843753815, "step": 85, "step_time": 22.43926484696567 }, { "clip_ratio/high_max": 0.001044186291983351, "clip_ratio/high_mean": 0.001044186291983351, "clip_ratio/low_mean": 0.00011646859929896891, "clip_ratio/low_min": 0.00011646859929896891, "clip_ratio/region_mean": 0.0011606548796407878, "completions/clipped_ratio": 0.0, "completions/max_length": 527.0, "completions/max_terminated_length": 527.0, "completions/mean_length": 310.6600036621094, "completions/mean_terminated_length": 310.6600036621094, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 0.14149793088436127, "epoch": 0.23369565217391305, "frac_reward_zero_std": 0.0, "grad_norm": 0.01278267614543438, "learning_rate": 5e-05, "loss": 0.01356387510895729, "num_tokens": 5385701.0, "reward": 0.8483105301856995, "reward_std": 0.046941135078668594, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.15168945491313934, "rewards/length_penalty/std": 0.04694113880395889, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9945849776268005, "sampling/importance_sampling_ratio/min": 0.2063879370689392, "sampling/sampling_logp_difference/max": 1.5779976844787598, "sampling/sampling_logp_difference/mean": 0.02506762556731701, "step": 86, "step_time": 6.345105629879981 }, { "clip_ratio/high_max": 0.000682604851317592, "clip_ratio/high_mean": 0.000682604851317592, "clip_ratio/low_mean": 9.332875779364259e-05, "clip_ratio/low_min": 9.332875779364259e-05, "clip_ratio/region_mean": 0.0007759336091112346, "completions/clipped_ratio": 0.0, "completions/max_length": 1127.0, "completions/max_terminated_length": 1127.0, "completions/mean_length": 498.41998291015625, "completions/mean_terminated_length": 498.41998291015625, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 0.14586662650108337, "epoch": 0.23641304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.0155003247782588, "learning_rate": 5e-05, "loss": 0.029746606945991516, "num_tokens": 5412832.0, "reward": 0.3566308617591858, "reward_std": 0.560200035572052, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.24336914718151093, "rewards/length_penalty/std": 0.12490728497505188, "sampling/importance_sampling_ratio/max": 2.270101308822632, "sampling/importance_sampling_ratio/mean": 0.9942162036895752, "sampling/importance_sampling_ratio/min": 0.2935054302215576, "sampling/sampling_logp_difference/max": 1.2258591651916504, "sampling/sampling_logp_difference/mean": 0.01999354176223278, "step": 87, "step_time": 13.222105751745403 }, { "clip_ratio/high_max": 0.0005343808559700846, "clip_ratio/high_mean": 0.0005343808559700846, "clip_ratio/low_mean": 0.00016299273702315986, "clip_ratio/low_min": 0.00016299273702315986, "clip_ratio/region_mean": 0.0006973735755309463, "completions/clipped_ratio": 0.0, "completions/max_length": 679.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 382.0799865722656, "completions/mean_terminated_length": 382.0799865722656, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 0.13788711428642272, "epoch": 0.2391304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.016201354563236237, "learning_rate": 5e-05, "loss": 0.008741471916437149, "num_tokens": 5434826.0, "reward": 0.7734375, "reward_std": 0.2171802967786789, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.18656249344348907, "rewards/length_penalty/std": 0.06984082609415054, "sampling/importance_sampling_ratio/max": 2.7883358001708984, "sampling/importance_sampling_ratio/mean": 0.9944828152656555, "sampling/importance_sampling_ratio/min": 0.1857496201992035, "sampling/sampling_logp_difference/max": 1.6833555698394775, "sampling/sampling_logp_difference/mean": 0.021422382444143295, "step": 88, "step_time": 8.28874902217649 }, { "clip_ratio/high_max": 0.0007955357184982858, "clip_ratio/high_mean": 0.0007955357184982858, "clip_ratio/low_mean": 2.621919265948236e-05, "clip_ratio/low_min": 2.621919265948236e-05, "clip_ratio/region_mean": 0.0008217549111577682, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1909.0, "completions/mean_length": 618.5, "completions/mean_terminated_length": 589.3265380859375, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.17175524830818176, "epoch": 0.2418478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.013653785921633244, "learning_rate": 5e-05, "loss": 0.020935919135808945, "num_tokens": 5469211.0, "reward": 0.21799804270267487, "reward_std": 0.6661584377288818, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.302001953125, "rewards/length_penalty/std": 0.2297671139240265, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930328130722046, "sampling/importance_sampling_ratio/min": 0.303615540266037, "sampling/sampling_logp_difference/max": 1.191992998123169, "sampling/sampling_logp_difference/mean": 0.020726434886455536, "step": 89, "step_time": 22.4951800100971 }, { "clip_ratio/high_max": 0.0007658500340767205, "clip_ratio/high_mean": 0.0007658500340767205, "clip_ratio/low_mean": 2.3438416246790438e-05, "clip_ratio/low_min": 2.3438416246790438e-05, "clip_ratio/region_mean": 0.0007892884546890855, "completions/clipped_ratio": 0.0, "completions/max_length": 1998.0, "completions/max_terminated_length": 1998.0, "completions/mean_length": 659.739990234375, "completions/mean_terminated_length": 659.739990234375, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.14435225278139113, "epoch": 0.24456521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.019286051392555237, "learning_rate": 5e-05, "loss": 0.03661634027957916, "num_tokens": 5505228.0, "reward": 0.6178613305091858, "reward_std": 0.36470702290534973, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979514837265, "rewards/length_penalty/mean": -0.3221386671066284, "rewards/length_penalty/std": 0.22656019032001495, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9943234920501709, "sampling/importance_sampling_ratio/min": 0.3591086268424988, "sampling/sampling_logp_difference/max": 1.1295256614685059, "sampling/sampling_logp_difference/mean": 0.019089285284280777, "step": 90, "step_time": 21.950008247047663 }, { "clip_ratio/high_max": 0.0009108542930334806, "clip_ratio/high_mean": 0.0009108542930334806, "clip_ratio/low_mean": 0.00010948879062198103, "clip_ratio/low_min": 0.00010948879062198103, "clip_ratio/region_mean": 0.0010203430894762278, "completions/clipped_ratio": 0.0, "completions/max_length": 1604.0, "completions/max_terminated_length": 1604.0, "completions/mean_length": 517.0800170898438, "completions/mean_terminated_length": 517.0800170898438, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "entropy": 0.17385907024145125, "epoch": 0.24728260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.017237825319170952, "learning_rate": 5e-05, "loss": 0.04393135383725166, "num_tokens": 5536252.0, "reward": 0.6875194907188416, "reward_std": 0.365161269903183, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979365825653, "rewards/length_penalty/mean": -0.25248047709465027, "rewards/length_penalty/std": 0.1945510357618332, "sampling/importance_sampling_ratio/max": 2.810533046722412, "sampling/importance_sampling_ratio/mean": 0.9928355813026428, "sampling/importance_sampling_ratio/min": 0.2914602756500244, "sampling/sampling_logp_difference/max": 1.232851505279541, "sampling/sampling_logp_difference/mean": 0.022177454084157944, "step": 91, "step_time": 18.435933465138078 }, { "clip_ratio/high_max": 0.00046310991747304796, "clip_ratio/high_mean": 0.00046310991747304796, "clip_ratio/low_mean": 0.00012931099627166986, "clip_ratio/low_min": 0.00012931099627166986, "clip_ratio/region_mean": 0.000592420919565484, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1596.0, "completions/mean_length": 511.2200012207031, "completions/mean_terminated_length": 479.8571472167969, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.14240270107984543, "epoch": 0.25, "frac_reward_zero_std": 0.0, "grad_norm": 0.014772976748645306, "learning_rate": 5e-05, "loss": 0.042228199541568756, "num_tokens": 5564203.0, "reward": 0.5503808259963989, "reward_std": 0.42025887966156006, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.24961914122104645, "rewards/length_penalty/std": 0.2091631144285202, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9942843317985535, "sampling/importance_sampling_ratio/min": 0.18323083221912384, "sampling/sampling_logp_difference/max": 1.6970086097717285, "sampling/sampling_logp_difference/mean": 0.021515555679798126, "step": 92, "step_time": 22.50430703512393 }, { "clip_ratio/high_max": 0.0007045186866889708, "clip_ratio/high_mean": 0.0007045186866889708, "clip_ratio/low_mean": 0.0002806035408866592, "clip_ratio/low_min": 0.0002806035408866592, "clip_ratio/region_mean": 0.0009851222421275452, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 1638.0, "completions/mean_length": 669.0999755859375, "completions/mean_terminated_length": 581.0850830078125, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "entropy": 0.20341256856918336, "epoch": 0.25271739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.020352164283394814, "learning_rate": 5e-05, "loss": 0.10169492661952972, "num_tokens": 5603278.0, "reward": 0.4732910096645355, "reward_std": 0.5773070454597473, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.32670897245407104, "rewards/length_penalty/std": 0.2420545071363449, "sampling/importance_sampling_ratio/max": 2.4024431705474854, "sampling/importance_sampling_ratio/mean": 0.9923534393310547, "sampling/importance_sampling_ratio/min": 0.2638809382915497, "sampling/sampling_logp_difference/max": 1.3322572708129883, "sampling/sampling_logp_difference/mean": 0.021952515468001366, "step": 93, "step_time": 23.539885837351903 }, { "clip_ratio/high_max": 0.0011616522213444113, "clip_ratio/high_mean": 0.0011616522213444113, "clip_ratio/low_mean": 0.0002181586460210383, "clip_ratio/low_min": 0.0002181586460210383, "clip_ratio/region_mean": 0.0013798108557239175, "completions/clipped_ratio": 0.0, "completions/max_length": 1176.0, "completions/max_terminated_length": 1176.0, "completions/mean_length": 450.239990234375, "completions/mean_terminated_length": 450.239990234375, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.19916799068450927, "epoch": 0.2554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.01750258356332779, "learning_rate": 5e-05, "loss": 0.0182659812271595, "num_tokens": 5631820.0, "reward": 0.30015623569488525, "reward_std": 0.461910218000412, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.21984374523162842, "rewards/length_penalty/std": 0.10487209260463715, "sampling/importance_sampling_ratio/max": 2.1481571197509766, "sampling/importance_sampling_ratio/mean": 0.991895854473114, "sampling/importance_sampling_ratio/min": 0.24272750318050385, "sampling/sampling_logp_difference/max": 1.415815830230713, "sampling/sampling_logp_difference/mean": 0.025588279590010643, "step": 94, "step_time": 13.953644986730069 }, { "clip_ratio/high_max": 0.0011509397649206222, "clip_ratio/high_mean": 0.0011509397649206222, "clip_ratio/low_mean": 0.00019863411434926093, "clip_ratio/low_min": 0.00019863411434926093, "clip_ratio/region_mean": 0.0013495738967321813, "completions/clipped_ratio": 0.0, "completions/max_length": 459.0, "completions/max_terminated_length": 459.0, "completions/mean_length": 290.0799865722656, "completions/mean_terminated_length": 290.0799865722656, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 0.1961733102798462, "epoch": 0.25815217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.01650562509894371, "learning_rate": 5e-05, "loss": 0.009883588179945946, "num_tokens": 5648694.0, "reward": 0.8383593559265137, "reward_std": 0.14430737495422363, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.14164061844348907, "rewards/length_penalty/std": 0.03877919539809227, "sampling/importance_sampling_ratio/max": 2.824411630630493, "sampling/importance_sampling_ratio/mean": 0.9931517839431763, "sampling/importance_sampling_ratio/min": 0.35756808519363403, "sampling/sampling_logp_difference/max": 1.0383000373840332, "sampling/sampling_logp_difference/mean": 0.030635839328169823, "step": 95, "step_time": 5.755662291543558 }, { "clip_ratio/high_max": 0.0006952405674383044, "clip_ratio/high_mean": 0.0006952405674383044, "clip_ratio/low_mean": 0.0003287168277893215, "clip_ratio/low_min": 0.0003287168277893215, "clip_ratio/region_mean": 0.0010239573777653277, "completions/clipped_ratio": 0.0, "completions/max_length": 684.0, "completions/max_terminated_length": 684.0, "completions/mean_length": 312.79998779296875, "completions/mean_terminated_length": 312.79998779296875, "completions/min_length": 101.0, "completions/min_terminated_length": 101.0, "entropy": 0.18830790221691132, "epoch": 0.2608695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.0161149799823761, "learning_rate": 5e-05, "loss": 0.014134292490780354, "num_tokens": 5667864.0, "reward": 0.5272656083106995, "reward_std": 0.5117128491401672, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.15273436903953552, "rewards/length_penalty/std": 0.0754011869430542, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9926605820655823, "sampling/importance_sampling_ratio/min": 0.2316138744354248, "sampling/sampling_logp_difference/max": 1.4626836776733398, "sampling/sampling_logp_difference/mean": 0.0293266698718071, "step": 96, "step_time": 8.08534470712766 }, { "clip_ratio/high_max": 0.0009365640406031161, "clip_ratio/high_mean": 0.0009365640406031161, "clip_ratio/low_mean": 7.815553108230233e-05, "clip_ratio/low_min": 7.815553108230233e-05, "clip_ratio/region_mean": 0.0010147195833269506, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 302.9599914550781, "completions/mean_terminated_length": 302.9599914550781, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 0.19039133489131926, "epoch": 0.26358695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.01323253195732832, "learning_rate": 5e-05, "loss": 0.007963886484503746, "num_tokens": 5685942.0, "reward": 0.8320702910423279, "reward_std": 0.14255400002002716, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.14792968332767487, "rewards/length_penalty/std": 0.053403496742248535, "sampling/importance_sampling_ratio/max": 2.9804489612579346, "sampling/importance_sampling_ratio/mean": 0.9915622472763062, "sampling/importance_sampling_ratio/min": 0.23956701159477234, "sampling/sampling_logp_difference/max": 1.428922176361084, "sampling/sampling_logp_difference/mean": 0.029504768550395966, "step": 97, "step_time": 7.233770550927147 }, { "clip_ratio/high_max": 0.0015867566457018257, "clip_ratio/high_mean": 0.0015867566457018257, "clip_ratio/low_mean": 0.00011946987069677562, "clip_ratio/low_min": 0.00011946987069677562, "clip_ratio/region_mean": 0.0017062265193089842, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 1352.0, "completions/mean_length": 349.7599792480469, "completions/mean_terminated_length": 241.3616943359375, "completions/min_length": 74.0, "completions/min_terminated_length": 74.0, "entropy": 0.22566365003585814, "epoch": 0.266304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.015148256905376911, "learning_rate": 5e-05, "loss": 0.0725148394703865, "num_tokens": 5705570.0, "reward": 0.2292187511920929, "reward_std": 0.6269381046295166, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.17078125476837158, "rewards/length_penalty/std": 0.24175436794757843, "sampling/importance_sampling_ratio/max": 2.5255091190338135, "sampling/importance_sampling_ratio/mean": 0.991366982460022, "sampling/importance_sampling_ratio/min": 0.09896191209554672, "sampling/sampling_logp_difference/max": 2.3130202293395996, "sampling/sampling_logp_difference/mean": 0.026597710326313972, "step": 98, "step_time": 21.33166946657002 }, { "clip_ratio/high_max": 0.0010735183954238892, "clip_ratio/high_mean": 0.0010735183954238892, "clip_ratio/low_mean": 0.00030840777326375247, "clip_ratio/low_min": 0.00030840777326375247, "clip_ratio/region_mean": 0.0013819261686876417, "completions/clipped_ratio": 0.0, "completions/max_length": 310.0, "completions/max_terminated_length": 310.0, "completions/mean_length": 185.86000061035156, "completions/mean_terminated_length": 185.86000061035156, "completions/min_length": 92.0, "completions/min_terminated_length": 92.0, "entropy": 0.20395126342773437, "epoch": 0.26902173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.012664223089814186, "learning_rate": 5e-05, "loss": 0.007070655468851328, "num_tokens": 5717773.0, "reward": 0.6292480230331421, "reward_std": 0.46103474497795105, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.09075195342302322, "rewards/length_penalty/std": 0.024525616317987442, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9933871626853943, "sampling/importance_sampling_ratio/min": 0.20495867729187012, "sampling/sampling_logp_difference/max": 1.584946870803833, "sampling/sampling_logp_difference/mean": 0.033781420439481735, "step": 99, "step_time": 4.117180567001924 }, { "clip_ratio/high_max": 0.0005707273143343627, "clip_ratio/high_mean": 0.0005707273143343627, "clip_ratio/low_mean": 9.760858956724406e-05, "clip_ratio/low_min": 9.760858956724406e-05, "clip_ratio/region_mean": 0.0006683359039016068, "completions/clipped_ratio": 0.0, "completions/max_length": 346.0, "completions/max_terminated_length": 346.0, "completions/mean_length": 187.25999450683594, "completions/mean_terminated_length": 187.25999450683594, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "entropy": 0.17016780972480774, "epoch": 0.2717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.012072809040546417, "learning_rate": 5e-05, "loss": 0.008675863966345787, "num_tokens": 5729366.0, "reward": 0.88856440782547, "reward_std": 0.14345325529575348, "rewards/correctness/mean": 0.9800000190734863, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.09143554419279099, "rewards/length_penalty/std": 0.03134666755795479, "sampling/importance_sampling_ratio/max": 2.592597246170044, "sampling/importance_sampling_ratio/mean": 0.991979718208313, "sampling/importance_sampling_ratio/min": 0.28697964549064636, "sampling/sampling_logp_difference/max": 1.2483439445495605, "sampling/sampling_logp_difference/mean": 0.03182890638709068, "step": 100, "step_time": 4.314533639000729 } ], "logging_steps": 1, "max_steps": 200, "num_input_tokens_seen": 5729366, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 10, "trial_name": null, "trial_params": null }