| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.4076086956521739, |
| "eval_steps": 500, |
| "global_step": 150, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.41999998688697815, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2017.0, |
| "completions/mean_length": 1671.2799072265625, |
| "completions/mean_terminated_length": 1398.4827880859375, |
| "completions/min_length": 992.0, |
| "completions/min_terminated_length": 992.0, |
| "entropy": 0.21248915791511536, |
| "epoch": 0.002717391304347826, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01622791402041912, |
| "learning_rate": 0.0, |
| "loss": 0.06695985794067383, |
| "num_tokens": 86304.0, |
| "reward": -0.21605467796325684, |
| "reward_std": 0.656523585319519, |
| "rewards/correctness/mean": 0.6000000238418579, |
| "rewards/correctness/std": 0.4948716461658478, |
| "rewards/length_penalty/mean": -0.8160547018051147, |
| "rewards/length_penalty/std": 0.18964654207229614, |
| "sampling/importance_sampling_ratio/max": 1.5222556591033936, |
| "sampling/importance_sampling_ratio/mean": 0.9927070140838623, |
| "sampling/importance_sampling_ratio/min": 0.6435883045196533, |
| "sampling/sampling_logp_difference/max": 0.44069600105285645, |
| "sampling/sampling_logp_difference/mean": 0.013988969847559929, |
| "step": 1, |
| "step_time": 27.388843874214217 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.6800000071525574, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1991.0, |
| "completions/mean_length": 1941.47998046875, |
| "completions/mean_terminated_length": 1715.125, |
| "completions/min_length": 1079.0, |
| "completions/min_terminated_length": 1079.0, |
| "entropy": 0.2905632793903351, |
| "epoch": 0.005434782608695652, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017553681507706642, |
| "learning_rate": 5e-06, |
| "loss": 0.06798752397298813, |
| "num_tokens": 186618.0, |
| "reward": -0.7279882431030273, |
| "reward_std": 0.4917677640914917, |
| "rewards/correctness/mean": 0.2199999988079071, |
| "rewards/correctness/std": 0.4184519350528717, |
| "rewards/length_penalty/mean": -0.9479882717132568, |
| "rewards/length_penalty/std": 0.10518474876880646, |
| "sampling/importance_sampling_ratio/max": 1.450933814048767, |
| "sampling/importance_sampling_ratio/mean": 0.9901063442230225, |
| "sampling/importance_sampling_ratio/min": 0.3688630163669586, |
| "sampling/sampling_logp_difference/max": 0.9973299503326416, |
| "sampling/sampling_logp_difference/mean": 0.01780563034117222, |
| "step": 2, |
| "step_time": 29.02393381111324 |
| }, |
| { |
| "clip_ratio/high_max": 0.0009701523347757756, |
| "clip_ratio/high_mean": 0.0009701523347757756, |
| "clip_ratio/low_mean": 6.219160495675169e-05, |
| "clip_ratio/low_min": 6.219160495675169e-05, |
| "clip_ratio/region_mean": 0.0010323439724743365, |
| "completions/clipped_ratio": 0.3199999928474426, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1901.0, |
| "completions/mean_length": 1612.3199462890625, |
| "completions/mean_terminated_length": 1407.2940673828125, |
| "completions/min_length": 745.0, |
| "completions/min_terminated_length": 745.0, |
| "entropy": 0.255071622133255, |
| "epoch": 0.008152173913043478, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017932165414094925, |
| "learning_rate": 1e-05, |
| "loss": 0.13346610963344574, |
| "num_tokens": 269504.0, |
| "reward": -0.08726562559604645, |
| "reward_std": 0.6200677156448364, |
| "rewards/correctness/mean": 0.699999988079071, |
| "rewards/correctness/std": 0.4629100263118744, |
| "rewards/length_penalty/mean": -0.7872655987739563, |
| "rewards/length_penalty/std": 0.19988179206848145, |
| "sampling/importance_sampling_ratio/max": 1.5381298065185547, |
| "sampling/importance_sampling_ratio/mean": 0.9911423921585083, |
| "sampling/importance_sampling_ratio/min": 0.6395388841629028, |
| "sampling/sampling_logp_difference/max": 0.4470078945159912, |
| "sampling/sampling_logp_difference/mean": 0.016371745616197586, |
| "step": 3, |
| "step_time": 27.040046802023426 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007077743241097779, |
| "clip_ratio/high_mean": 0.0007077743241097779, |
| "clip_ratio/low_mean": 6.867761694593356e-05, |
| "clip_ratio/low_min": 6.867761694593356e-05, |
| "clip_ratio/region_mean": 0.0007764519308693707, |
| "completions/clipped_ratio": 0.29999998211860657, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1882.0, |
| "completions/mean_length": 1489.419921875, |
| "completions/mean_terminated_length": 1250.028564453125, |
| "completions/min_length": 626.0, |
| "completions/min_terminated_length": 626.0, |
| "entropy": 0.24563565850257874, |
| "epoch": 0.010869565217391304, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017823319882154465, |
| "learning_rate": 1.5e-05, |
| "loss": 0.09811276942491531, |
| "num_tokens": 346875.0, |
| "reward": -0.007255859207361937, |
| "reward_std": 0.6479107141494751, |
| "rewards/correctness/mean": 0.7200000286102295, |
| "rewards/correctness/std": 0.4535573720932007, |
| "rewards/length_penalty/mean": -0.7272558808326721, |
| "rewards/length_penalty/std": 0.2412886619567871, |
| "sampling/importance_sampling_ratio/max": 1.5703521966934204, |
| "sampling/importance_sampling_ratio/mean": 0.9914191961288452, |
| "sampling/importance_sampling_ratio/min": 0.5496033430099487, |
| "sampling/sampling_logp_difference/max": 0.5985584259033203, |
| "sampling/sampling_logp_difference/mean": 0.016086140647530556, |
| "step": 4, |
| "step_time": 27.06473581609316 |
| }, |
| { |
| "clip_ratio/high_max": 0.0014693949837237597, |
| "clip_ratio/high_mean": 0.0014693949837237597, |
| "clip_ratio/low_mean": 2.9735209682257847e-05, |
| "clip_ratio/low_min": 2.9735209682257847e-05, |
| "clip_ratio/region_mean": 0.0014991301926784218, |
| "completions/clipped_ratio": 0.8199999928474426, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1909.0, |
| "completions/mean_length": 1955.3800048828125, |
| "completions/mean_terminated_length": 1533.4444580078125, |
| "completions/min_length": 1097.0, |
| "completions/min_terminated_length": 1097.0, |
| "entropy": 0.2879446566104889, |
| "epoch": 0.01358695652173913, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.015369652770459652, |
| "learning_rate": 2e-05, |
| "loss": 0.05972106009721756, |
| "num_tokens": 448354.0, |
| "reward": -0.7747753858566284, |
| "reward_std": 0.4882567822933197, |
| "rewards/correctness/mean": 0.18000000715255737, |
| "rewards/correctness/std": 0.3880879282951355, |
| "rewards/length_penalty/mean": -0.9547753930091858, |
| "rewards/length_penalty/std": 0.11000200361013412, |
| "sampling/importance_sampling_ratio/max": 2.131683349609375, |
| "sampling/importance_sampling_ratio/mean": 0.9901448488235474, |
| "sampling/importance_sampling_ratio/min": 0.6602146625518799, |
| "sampling/sampling_logp_difference/max": 0.7569119930267334, |
| "sampling/sampling_logp_difference/mean": 0.01812317781150341, |
| "step": 5, |
| "step_time": 29.137280595256016 |
| }, |
| { |
| "clip_ratio/high_max": 0.000520571404194925, |
| "clip_ratio/high_mean": 0.000520571404194925, |
| "clip_ratio/low_mean": 0.00017908125300891697, |
| "clip_ratio/low_min": 0.00017908125300891697, |
| "clip_ratio/region_mean": 0.0006996526441071183, |
| "completions/clipped_ratio": 0.4599999785423279, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1941.0, |
| "completions/mean_length": 1660.0, |
| "completions/mean_terminated_length": 1329.4814453125, |
| "completions/min_length": 723.0, |
| "completions/min_terminated_length": 723.0, |
| "entropy": 0.2539885014295578, |
| "epoch": 0.016304347826086956, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01736672781407833, |
| "learning_rate": 2.5e-05, |
| "loss": 0.08885529637336731, |
| "num_tokens": 534344.0, |
| "reward": -0.23054687678813934, |
| "reward_std": 0.6794751286506653, |
| "rewards/correctness/mean": 0.5799999833106995, |
| "rewards/correctness/std": 0.49856939911842346, |
| "rewards/length_penalty/mean": -0.810546875, |
| "rewards/length_penalty/std": 0.22522568702697754, |
| "sampling/importance_sampling_ratio/max": 1.742319941520691, |
| "sampling/importance_sampling_ratio/mean": 0.9909853339195251, |
| "sampling/importance_sampling_ratio/min": 0.5412515997886658, |
| "sampling/sampling_logp_difference/max": 0.6138710975646973, |
| "sampling/sampling_logp_difference/mean": 0.016689306125044823, |
| "step": 6, |
| "step_time": 27.337108821840957 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006314931088127196, |
| "clip_ratio/high_mean": 0.0006314931088127196, |
| "clip_ratio/low_mean": 7.634440262336284e-05, |
| "clip_ratio/low_min": 7.634440262336284e-05, |
| "clip_ratio/region_mean": 0.0007078375143464655, |
| "completions/clipped_ratio": 0.29999998211860657, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2029.0, |
| "completions/mean_length": 1579.8399658203125, |
| "completions/mean_terminated_length": 1379.199951171875, |
| "completions/min_length": 803.0, |
| "completions/min_terminated_length": 803.0, |
| "entropy": 0.2385493516921997, |
| "epoch": 0.019021739130434784, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.016621841117739677, |
| "learning_rate": 3e-05, |
| "loss": 0.08834289014339447, |
| "num_tokens": 615566.0, |
| "reward": -0.05140624940395355, |
| "reward_std": 0.6118109226226807, |
| "rewards/correctness/mean": 0.7200000286102295, |
| "rewards/correctness/std": 0.4535573720932007, |
| "rewards/length_penalty/mean": -0.7714062333106995, |
| "rewards/length_penalty/std": 0.19486749172210693, |
| "sampling/importance_sampling_ratio/max": 1.4593793153762817, |
| "sampling/importance_sampling_ratio/mean": 0.9915305972099304, |
| "sampling/importance_sampling_ratio/min": 0.651093065738678, |
| "sampling/sampling_logp_difference/max": 0.42910265922546387, |
| "sampling/sampling_logp_difference/mean": 0.015702536329627037, |
| "step": 7, |
| "step_time": 27.038635928882286 |
| }, |
| { |
| "clip_ratio/high_max": 0.0008715830335859209, |
| "clip_ratio/high_mean": 0.0008715830335859209, |
| "clip_ratio/low_mean": 0.00011999263515463098, |
| "clip_ratio/low_min": 0.00011999263515463098, |
| "clip_ratio/region_mean": 0.0009915756527334452, |
| "completions/clipped_ratio": 0.5999999642372131, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1938.0, |
| "completions/mean_length": 1801.239990234375, |
| "completions/mean_terminated_length": 1431.0999755859375, |
| "completions/min_length": 930.0, |
| "completions/min_terminated_length": 930.0, |
| "entropy": 0.21174302995204924, |
| "epoch": 0.021739130434782608, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01676737703382969, |
| "learning_rate": 3.5e-05, |
| "loss": 0.08332858979701996, |
| "num_tokens": 707778.0, |
| "reward": -0.47951170802116394, |
| "reward_std": 0.6519015431404114, |
| "rewards/correctness/mean": 0.4000000059604645, |
| "rewards/correctness/std": 0.4948716461658478, |
| "rewards/length_penalty/mean": -0.8795117139816284, |
| "rewards/length_penalty/std": 0.18039129674434662, |
| "sampling/importance_sampling_ratio/max": 1.4509235620498657, |
| "sampling/importance_sampling_ratio/mean": 0.9925462007522583, |
| "sampling/importance_sampling_ratio/min": 0.052912116050720215, |
| "sampling/sampling_logp_difference/max": 2.9391229152679443, |
| "sampling/sampling_logp_difference/mean": 0.014070631936192513, |
| "step": 8, |
| "step_time": 28.223328100983053 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006956349650863558, |
| "clip_ratio/high_mean": 0.0006956349650863558, |
| "clip_ratio/low_mean": 8.642674874863588e-05, |
| "clip_ratio/low_min": 8.642674874863588e-05, |
| "clip_ratio/region_mean": 0.0007820617058314383, |
| "completions/clipped_ratio": 0.25999999046325684, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1994.0, |
| "completions/mean_length": 1600.239990234375, |
| "completions/mean_terminated_length": 1442.9189453125, |
| "completions/min_length": 785.0, |
| "completions/min_terminated_length": 785.0, |
| "entropy": 0.23202865421772004, |
| "epoch": 0.024456521739130436, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017678245902061462, |
| "learning_rate": 4e-05, |
| "loss": 0.09870695322751999, |
| "num_tokens": 790260.0, |
| "reward": -0.20136718451976776, |
| "reward_std": 0.6241466403007507, |
| "rewards/correctness/mean": 0.5799999833106995, |
| "rewards/correctness/std": 0.49856939911842346, |
| "rewards/length_penalty/mean": -0.7813671827316284, |
| "rewards/length_penalty/std": 0.1977044939994812, |
| "sampling/importance_sampling_ratio/max": 1.5457746982574463, |
| "sampling/importance_sampling_ratio/mean": 0.991839587688446, |
| "sampling/importance_sampling_ratio/min": 0.556106448173523, |
| "sampling/sampling_logp_difference/max": 0.5867955684661865, |
| "sampling/sampling_logp_difference/mean": 0.01518256589770317, |
| "step": 9, |
| "step_time": 27.120734592201188 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007978752721101046, |
| "clip_ratio/high_mean": 0.0007978752721101046, |
| "clip_ratio/low_mean": 7.057307666400448e-05, |
| "clip_ratio/low_min": 7.057307666400448e-05, |
| "clip_ratio/region_mean": 0.0008684483473189175, |
| "completions/clipped_ratio": 0.2800000011920929, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2048.0, |
| "completions/mean_length": 1484.0999755859375, |
| "completions/mean_terminated_length": 1264.8055419921875, |
| "completions/min_length": 668.0, |
| "completions/min_terminated_length": 668.0, |
| "entropy": 0.24629398584365844, |
| "epoch": 0.02717391304347826, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017371023073792458, |
| "learning_rate": 4.5e-05, |
| "loss": 0.09812771528959274, |
| "num_tokens": 869285.0, |
| "reward": -0.024658203125, |
| "reward_std": 0.647922933101654, |
| "rewards/correctness/mean": 0.699999988079071, |
| "rewards/correctness/std": 0.4629100263118744, |
| "rewards/length_penalty/mean": -0.724658191204071, |
| "rewards/length_penalty/std": 0.2301420420408249, |
| "sampling/importance_sampling_ratio/max": 1.4670653343200684, |
| "sampling/importance_sampling_ratio/mean": 0.9915652275085449, |
| "sampling/importance_sampling_ratio/min": 0.6655772924423218, |
| "sampling/sampling_logp_difference/max": 0.4071004390716553, |
| "sampling/sampling_logp_difference/mean": 0.015878628939390182, |
| "step": 10, |
| "step_time": 27.66670753993094 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007490424031857401, |
| "clip_ratio/high_mean": 0.0007490424031857401, |
| "clip_ratio/low_mean": 0.00011397197667974978, |
| "clip_ratio/low_min": 0.00011397197667974978, |
| "clip_ratio/region_mean": 0.0008630143827758729, |
| "completions/clipped_ratio": 0.47999998927116394, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2034.0, |
| "completions/mean_length": 1854.7999267578125, |
| "completions/mean_terminated_length": 1676.4615478515625, |
| "completions/min_length": 1178.0, |
| "completions/min_terminated_length": 1178.0, |
| "entropy": 0.24558066129684447, |
| "epoch": 0.029891304347826088, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01811002753674984, |
| "learning_rate": 5e-05, |
| "loss": 0.08611460030078888, |
| "num_tokens": 964445.0, |
| "reward": -0.32566404342651367, |
| "reward_std": 0.5863887667655945, |
| "rewards/correctness/mean": 0.5799999833106995, |
| "rewards/correctness/std": 0.49856939911842346, |
| "rewards/length_penalty/mean": -0.9056640863418579, |
| "rewards/length_penalty/std": 0.12008737027645111, |
| "sampling/importance_sampling_ratio/max": 1.515191912651062, |
| "sampling/importance_sampling_ratio/mean": 0.9914591312408447, |
| "sampling/importance_sampling_ratio/min": 0.2949647009372711, |
| "sampling/sampling_logp_difference/max": 1.2208995819091797, |
| "sampling/sampling_logp_difference/mean": 0.0158162210136652, |
| "step": 11, |
| "step_time": 28.245034573134035 |
| }, |
| { |
| "clip_ratio/high_max": 0.0004984279570635408, |
| "clip_ratio/high_mean": 0.0004984279570635408, |
| "clip_ratio/low_mean": 0.00014867318095639347, |
| "clip_ratio/low_min": 0.00014867318095639347, |
| "clip_ratio/region_mean": 0.0006471011089161039, |
| "completions/clipped_ratio": 0.5, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2027.0, |
| "completions/mean_length": 1772.43994140625, |
| "completions/mean_terminated_length": 1496.8800048828125, |
| "completions/min_length": 1000.0, |
| "completions/min_terminated_length": 1000.0, |
| "entropy": 0.2841207951307297, |
| "epoch": 0.03260869565217391, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.018444733694195747, |
| "learning_rate": 5e-05, |
| "loss": 0.10594244301319122, |
| "num_tokens": 1055707.0, |
| "reward": -0.3654492199420929, |
| "reward_std": 0.6480823159217834, |
| "rewards/correctness/mean": 0.5, |
| "rewards/correctness/std": 0.5050762891769409, |
| "rewards/length_penalty/mean": -0.8654491901397705, |
| "rewards/length_penalty/std": 0.1661682426929474, |
| "sampling/importance_sampling_ratio/max": 1.595076084136963, |
| "sampling/importance_sampling_ratio/mean": 0.9902372360229492, |
| "sampling/importance_sampling_ratio/min": 0.6496405005455017, |
| "sampling/sampling_logp_difference/max": 0.46692144870758057, |
| "sampling/sampling_logp_difference/mean": 0.018158787861466408, |
| "step": 12, |
| "step_time": 28.327091000042856 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005573231639573351, |
| "clip_ratio/high_mean": 0.0005573231639573351, |
| "clip_ratio/low_mean": 4.4240543502382934e-05, |
| "clip_ratio/low_min": 4.4240543502382934e-05, |
| "clip_ratio/region_mean": 0.0006015637016389519, |
| "completions/clipped_ratio": 0.3999999761581421, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1992.0, |
| "completions/mean_length": 1732.1199951171875, |
| "completions/mean_terminated_length": 1521.533447265625, |
| "completions/min_length": 840.0, |
| "completions/min_terminated_length": 840.0, |
| "entropy": 0.25946328043937683, |
| "epoch": 0.035326086956521736, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01701098121702671, |
| "learning_rate": 5e-05, |
| "loss": 0.07992903143167496, |
| "num_tokens": 1145333.0, |
| "reward": -0.40576171875, |
| "reward_std": 0.6230165958404541, |
| "rewards/correctness/mean": 0.4399999976158142, |
| "rewards/correctness/std": 0.5014265179634094, |
| "rewards/length_penalty/mean": -0.8457617163658142, |
| "rewards/length_penalty/std": 0.18863798677921295, |
| "sampling/importance_sampling_ratio/max": 1.4509283304214478, |
| "sampling/importance_sampling_ratio/mean": 0.9908791780471802, |
| "sampling/importance_sampling_ratio/min": 0.6448545455932617, |
| "sampling/sampling_logp_difference/max": 0.43873047828674316, |
| "sampling/sampling_logp_difference/mean": 0.016880951821804047, |
| "step": 13, |
| "step_time": 27.769306767731905 |
| }, |
| { |
| "clip_ratio/high_max": 0.000875298772007227, |
| "clip_ratio/high_mean": 0.000875298772007227, |
| "clip_ratio/low_mean": 0.00014390671713044868, |
| "clip_ratio/low_min": 0.00014390671713044868, |
| "clip_ratio/region_mean": 0.001019205478951335, |
| "completions/clipped_ratio": 0.47999998927116394, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2015.0, |
| "completions/mean_length": 1752.8399658203125, |
| "completions/mean_terminated_length": 1480.3846435546875, |
| "completions/min_length": 975.0, |
| "completions/min_terminated_length": 975.0, |
| "entropy": 0.2478518307209015, |
| "epoch": 0.03804347826086957, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017872340977191925, |
| "learning_rate": 5e-05, |
| "loss": 0.10336586833000183, |
| "num_tokens": 1235535.0, |
| "reward": -0.3358789086341858, |
| "reward_std": 0.654521644115448, |
| "rewards/correctness/mean": 0.5199999809265137, |
| "rewards/correctness/std": 0.5046720504760742, |
| "rewards/length_penalty/mean": -0.8558788895606995, |
| "rewards/length_penalty/std": 0.18034636974334717, |
| "sampling/importance_sampling_ratio/max": 1.4315266609191895, |
| "sampling/importance_sampling_ratio/mean": 0.9911905527114868, |
| "sampling/importance_sampling_ratio/min": 0.5119919180870056, |
| "sampling/sampling_logp_difference/max": 0.6694464683532715, |
| "sampling/sampling_logp_difference/mean": 0.016088664531707764, |
| "step": 14, |
| "step_time": 28.357874361332506 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006910582887940109, |
| "clip_ratio/high_mean": 0.0006910582887940109, |
| "clip_ratio/low_mean": 8.928978495532647e-05, |
| "clip_ratio/low_min": 8.928978495532647e-05, |
| "clip_ratio/region_mean": 0.0007803480722941459, |
| "completions/clipped_ratio": 0.2800000011920929, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1903.0, |
| "completions/mean_length": 1541.0399169921875, |
| "completions/mean_terminated_length": 1343.888916015625, |
| "completions/min_length": 961.0, |
| "completions/min_terminated_length": 961.0, |
| "entropy": 0.17789481580257416, |
| "epoch": 0.04076086956521739, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.019630515947937965, |
| "learning_rate": 5e-05, |
| "loss": 0.0840536430478096, |
| "num_tokens": 1315167.0, |
| "reward": -0.2124609351158142, |
| "reward_std": 0.5916866064071655, |
| "rewards/correctness/mean": 0.5400000214576721, |
| "rewards/correctness/std": 0.5034574270248413, |
| "rewards/length_penalty/mean": -0.7524609565734863, |
| "rewards/length_penalty/std": 0.18993473052978516, |
| "sampling/importance_sampling_ratio/max": 1.4382117986679077, |
| "sampling/importance_sampling_ratio/mean": 0.9938375353813171, |
| "sampling/importance_sampling_ratio/min": 0.6220142841339111, |
| "sampling/sampling_logp_difference/max": 0.4747922420501709, |
| "sampling/sampling_logp_difference/mean": 0.011926891282200813, |
| "step": 15, |
| "step_time": 27.46101871319115 |
| }, |
| { |
| "clip_ratio/high_max": 0.0012169468565844, |
| "clip_ratio/high_mean": 0.0012169468565844, |
| "clip_ratio/low_mean": 6.13222160609439e-05, |
| "clip_ratio/low_min": 6.13222160609439e-05, |
| "clip_ratio/region_mean": 0.0012782690697349607, |
| "completions/clipped_ratio": 0.41999998688697815, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1896.0, |
| "completions/mean_length": 1608.760009765625, |
| "completions/mean_terminated_length": 1290.689697265625, |
| "completions/min_length": 763.0, |
| "completions/min_terminated_length": 763.0, |
| "entropy": 0.25876877307891843, |
| "epoch": 0.043478260869565216, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01870792917907238, |
| "learning_rate": 5e-05, |
| "loss": 0.10105059295892715, |
| "num_tokens": 1399705.0, |
| "reward": -0.18552733957767487, |
| "reward_std": 0.6857022643089294, |
| "rewards/correctness/mean": 0.6000000238418579, |
| "rewards/correctness/std": 0.4948716461658478, |
| "rewards/length_penalty/mean": -0.7855273485183716, |
| "rewards/length_penalty/std": 0.22407564520835876, |
| "sampling/importance_sampling_ratio/max": 1.8998873233795166, |
| "sampling/importance_sampling_ratio/mean": 0.990841805934906, |
| "sampling/importance_sampling_ratio/min": 0.5615923404693604, |
| "sampling/sampling_logp_difference/max": 0.6417945623397827, |
| "sampling/sampling_logp_difference/mean": 0.016635797917842865, |
| "step": 16, |
| "step_time": 27.157737250905484 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006164561185869388, |
| "clip_ratio/high_mean": 0.0006164561185869388, |
| "clip_ratio/low_mean": 0.0001622869363927748, |
| "clip_ratio/low_min": 0.0001622869363927748, |
| "clip_ratio/region_mean": 0.0007787430338794365, |
| "completions/clipped_ratio": 0.47999998927116394, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2028.0, |
| "completions/mean_length": 1708.8399658203125, |
| "completions/mean_terminated_length": 1395.769287109375, |
| "completions/min_length": 868.0, |
| "completions/min_terminated_length": 868.0, |
| "entropy": 0.2281492084264755, |
| "epoch": 0.04619565217391304, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01940748654305935, |
| "learning_rate": 5e-05, |
| "loss": 0.07547423243522644, |
| "num_tokens": 1487797.0, |
| "reward": -0.3143945336341858, |
| "reward_std": 0.6761555075645447, |
| "rewards/correctness/mean": 0.5199999809265137, |
| "rewards/correctness/std": 0.5046720504760742, |
| "rewards/length_penalty/mean": -0.8343945145606995, |
| "rewards/length_penalty/std": 0.20066553354263306, |
| "sampling/importance_sampling_ratio/max": 1.667822241783142, |
| "sampling/importance_sampling_ratio/mean": 0.9918660521507263, |
| "sampling/importance_sampling_ratio/min": 0.5456497669219971, |
| "sampling/sampling_logp_difference/max": 0.6057779788970947, |
| "sampling/sampling_logp_difference/mean": 0.01505908090621233, |
| "step": 17, |
| "step_time": 27.55593749205582 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007688433746807277, |
| "clip_ratio/high_mean": 0.0007688433746807277, |
| "clip_ratio/low_mean": 6.974122588871978e-05, |
| "clip_ratio/low_min": 6.974122588871978e-05, |
| "clip_ratio/region_mean": 0.0008385846158489585, |
| "completions/clipped_ratio": 0.23999999463558197, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2045.0, |
| "completions/mean_length": 1681.239990234375, |
| "completions/mean_terminated_length": 1565.4210205078125, |
| "completions/min_length": 1013.0, |
| "completions/min_terminated_length": 1013.0, |
| "entropy": 0.2069709062576294, |
| "epoch": 0.04891304347826087, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01836375705897808, |
| "learning_rate": 5e-05, |
| "loss": 0.07770835608243942, |
| "num_tokens": 1574359.0, |
| "reward": -0.20091795921325684, |
| "reward_std": 0.6101290583610535, |
| "rewards/correctness/mean": 0.6200000047683716, |
| "rewards/correctness/std": 0.4903143644332886, |
| "rewards/length_penalty/mean": -0.8209179639816284, |
| "rewards/length_penalty/std": 0.1540166735649109, |
| "sampling/importance_sampling_ratio/max": 1.4398912191390991, |
| "sampling/importance_sampling_ratio/mean": 0.9928213357925415, |
| "sampling/importance_sampling_ratio/min": 0.6220183372497559, |
| "sampling/sampling_logp_difference/max": 0.4747856855392456, |
| "sampling/sampling_logp_difference/mean": 0.013566325418651104, |
| "step": 18, |
| "step_time": 27.951239524874836 |
| }, |
| { |
| "clip_ratio/high_max": 0.0014653601916506886, |
| "clip_ratio/high_mean": 0.0014653601916506886, |
| "clip_ratio/low_mean": 0.00016244076541624964, |
| "clip_ratio/low_min": 0.00016244076541624964, |
| "clip_ratio/region_mean": 0.0016278009628877045, |
| "completions/clipped_ratio": 0.5600000023841858, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2028.0, |
| "completions/mean_length": 1752.8199462890625, |
| "completions/mean_terminated_length": 1377.1363525390625, |
| "completions/min_length": 589.0, |
| "completions/min_terminated_length": 589.0, |
| "entropy": 0.2843230068683624, |
| "epoch": 0.051630434782608696, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.019417675212025642, |
| "learning_rate": 5e-05, |
| "loss": 0.0901256799697876, |
| "num_tokens": 1665430.0, |
| "reward": -0.3958691358566284, |
| "reward_std": 0.6787852048873901, |
| "rewards/correctness/mean": 0.46000000834465027, |
| "rewards/correctness/std": 0.5034573674201965, |
| "rewards/length_penalty/mean": -0.8558691143989563, |
| "rewards/length_penalty/std": 0.22009503841400146, |
| "sampling/importance_sampling_ratio/max": 1.4385100603103638, |
| "sampling/importance_sampling_ratio/mean": 0.9899238348007202, |
| "sampling/importance_sampling_ratio/min": 0.5758324265480042, |
| "sampling/sampling_logp_difference/max": 0.5519386529922485, |
| "sampling/sampling_logp_difference/mean": 0.0180143304169178, |
| "step": 19, |
| "step_time": 28.49813059018925 |
| }, |
| { |
| "clip_ratio/high_max": 0.0012508025160059334, |
| "clip_ratio/high_mean": 0.0012508025160059334, |
| "clip_ratio/low_mean": 5.348440172383562e-05, |
| "clip_ratio/low_min": 5.348440172383562e-05, |
| "clip_ratio/region_mean": 0.0013042868813499808, |
| "completions/clipped_ratio": 0.5199999809265137, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2046.0, |
| "completions/mean_length": 1844.43994140625, |
| "completions/mean_terminated_length": 1623.916748046875, |
| "completions/min_length": 1120.0, |
| "completions/min_terminated_length": 1120.0, |
| "entropy": 0.2606961756944656, |
| "epoch": 0.05434782608695652, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01755714789032936, |
| "learning_rate": 5e-05, |
| "loss": 0.09018535166978836, |
| "num_tokens": 1760242.0, |
| "reward": -0.42060545086860657, |
| "reward_std": 0.6140408515930176, |
| "rewards/correctness/mean": 0.47999998927116394, |
| "rewards/correctness/std": 0.5046720504760742, |
| "rewards/length_penalty/mean": -0.9006054401397705, |
| "rewards/length_penalty/std": 0.12989383935928345, |
| "sampling/importance_sampling_ratio/max": 1.3885498046875, |
| "sampling/importance_sampling_ratio/mean": 0.9910284280776978, |
| "sampling/importance_sampling_ratio/min": 0.4727957248687744, |
| "sampling/sampling_logp_difference/max": 0.7490918636322021, |
| "sampling/sampling_logp_difference/mean": 0.01643785461783409, |
| "step": 20, |
| "step_time": 27.96463798894547 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007326291175559163, |
| "clip_ratio/high_mean": 0.0007326291175559163, |
| "clip_ratio/low_mean": 0.00013991183222969994, |
| "clip_ratio/low_min": 0.00013991183222969994, |
| "clip_ratio/region_mean": 0.0008725409512408078, |
| "completions/clipped_ratio": 0.29999998211860657, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2016.0, |
| "completions/mean_length": 1689.5599365234375, |
| "completions/mean_terminated_length": 1535.94287109375, |
| "completions/min_length": 1011.0, |
| "completions/min_terminated_length": 1011.0, |
| "entropy": 0.2159762352705002, |
| "epoch": 0.057065217391304345, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.016861431300640106, |
| "learning_rate": 5e-05, |
| "loss": 0.08257238566875458, |
| "num_tokens": 1848130.0, |
| "reward": -0.12498046457767487, |
| "reward_std": 0.5931032299995422, |
| "rewards/correctness/mean": 0.699999988079071, |
| "rewards/correctness/std": 0.4629100263118744, |
| "rewards/length_penalty/mean": -0.8249804973602295, |
| "rewards/length_penalty/std": 0.17415757477283478, |
| "sampling/importance_sampling_ratio/max": 1.4638551473617554, |
| "sampling/importance_sampling_ratio/mean": 0.9920001029968262, |
| "sampling/importance_sampling_ratio/min": 0.5236321091651917, |
| "sampling/sampling_logp_difference/max": 0.6469658613204956, |
| "sampling/sampling_logp_difference/mean": 0.014182405546307564, |
| "step": 21, |
| "step_time": 26.797411655075848 |
| }, |
| { |
| "clip_ratio/high_max": 0.0011293109448160977, |
| "clip_ratio/high_mean": 0.0011293109448160977, |
| "clip_ratio/low_mean": 9.711003804113716e-05, |
| "clip_ratio/low_min": 9.711003804113716e-05, |
| "clip_ratio/region_mean": 0.0012264209799468517, |
| "completions/clipped_ratio": 0.2800000011920929, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1946.0, |
| "completions/mean_length": 1501.0599365234375, |
| "completions/mean_terminated_length": 1288.361083984375, |
| "completions/min_length": 687.0, |
| "completions/min_terminated_length": 687.0, |
| "entropy": 0.2569431811571121, |
| "epoch": 0.059782608695652176, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.020697707310318947, |
| "learning_rate": 5e-05, |
| "loss": 0.12516915798187256, |
| "num_tokens": 1928253.0, |
| "reward": 0.0070605468936264515, |
| "reward_std": 0.6210074424743652, |
| "rewards/correctness/mean": 0.7400000095367432, |
| "rewards/correctness/std": 0.44308751821517944, |
| "rewards/length_penalty/mean": -0.7329394817352295, |
| "rewards/length_penalty/std": 0.21821558475494385, |
| "sampling/importance_sampling_ratio/max": 1.6245068311691284, |
| "sampling/importance_sampling_ratio/mean": 0.9910734295845032, |
| "sampling/importance_sampling_ratio/min": 0.5689723491668701, |
| "sampling/sampling_logp_difference/max": 0.5639233589172363, |
| "sampling/sampling_logp_difference/mean": 0.016499562188982964, |
| "step": 22, |
| "step_time": 27.23643856518902 |
| }, |
| { |
| "clip_ratio/high_max": 0.0009234562981873751, |
| "clip_ratio/high_mean": 0.0009234562981873751, |
| "clip_ratio/low_mean": 0.00011172640952281654, |
| "clip_ratio/low_min": 0.00011172640952281654, |
| "clip_ratio/region_mean": 0.0010351826902478933, |
| "completions/clipped_ratio": 0.3400000035762787, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1951.0, |
| "completions/mean_length": 1656.260009765625, |
| "completions/mean_terminated_length": 1454.45458984375, |
| "completions/min_length": 875.0, |
| "completions/min_terminated_length": 875.0, |
| "entropy": 0.2721762388944626, |
| "epoch": 0.0625, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02058061957359314, |
| "learning_rate": 5e-05, |
| "loss": 0.1155262216925621, |
| "num_tokens": 2017186.0, |
| "reward": -0.2487206906080246, |
| "reward_std": 0.6356157064437866, |
| "rewards/correctness/mean": 0.5600000023841858, |
| "rewards/correctness/std": 0.5014265179634094, |
| "rewards/length_penalty/mean": -0.8087207078933716, |
| "rewards/length_penalty/std": 0.1684350073337555, |
| "sampling/importance_sampling_ratio/max": 1.5564347505569458, |
| "sampling/importance_sampling_ratio/mean": 0.9903766512870789, |
| "sampling/importance_sampling_ratio/min": 0.5593669414520264, |
| "sampling/sampling_logp_difference/max": 0.5809495449066162, |
| "sampling/sampling_logp_difference/mean": 0.01754668541252613, |
| "step": 23, |
| "step_time": 27.693883403902873 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005631370135233737, |
| "clip_ratio/high_mean": 0.0005631370135233737, |
| "clip_ratio/low_mean": 0.00010890306512010284, |
| "clip_ratio/low_min": 0.00010890306512010284, |
| "clip_ratio/region_mean": 0.0006720400706399232, |
| "completions/clipped_ratio": 0.41999998688697815, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2034.0, |
| "completions/mean_length": 1659.1199951171875, |
| "completions/mean_terminated_length": 1377.5172119140625, |
| "completions/min_length": 734.0, |
| "completions/min_terminated_length": 734.0, |
| "entropy": 0.2400749534368515, |
| "epoch": 0.06521739130434782, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017921488732099533, |
| "learning_rate": 5e-05, |
| "loss": 0.0665205717086792, |
| "num_tokens": 2102432.0, |
| "reward": -0.2701171934604645, |
| "reward_std": 0.675529420375824, |
| "rewards/correctness/mean": 0.5400000214576721, |
| "rewards/correctness/std": 0.5034574270248413, |
| "rewards/length_penalty/mean": -0.8101171851158142, |
| "rewards/length_penalty/std": 0.22776594758033752, |
| "sampling/importance_sampling_ratio/max": 1.6843202114105225, |
| "sampling/importance_sampling_ratio/mean": 0.9915715456008911, |
| "sampling/importance_sampling_ratio/min": 0.4471953809261322, |
| "sampling/sampling_logp_difference/max": 0.8047597408294678, |
| "sampling/sampling_logp_difference/mean": 0.01562676392495632, |
| "step": 24, |
| "step_time": 27.030204062117264 |
| }, |
| { |
| "clip_ratio/high_max": 0.0010878795757889748, |
| "clip_ratio/high_mean": 0.0010878795757889748, |
| "clip_ratio/low_mean": 0.00010777307761600241, |
| "clip_ratio/low_min": 0.00010777307761600241, |
| "clip_ratio/region_mean": 0.0011956526315771044, |
| "completions/clipped_ratio": 0.2199999988079071, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2004.0, |
| "completions/mean_length": 1457.0599365234375, |
| "completions/mean_terminated_length": 1290.3846435546875, |
| "completions/min_length": 754.0, |
| "completions/min_terminated_length": 754.0, |
| "entropy": 0.28386659622192384, |
| "epoch": 0.06793478260869565, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.021549172699451447, |
| "learning_rate": 5e-05, |
| "loss": 0.10626962780952454, |
| "num_tokens": 2177685.0, |
| "reward": 0.04854492098093033, |
| "reward_std": 0.6079964637756348, |
| "rewards/correctness/mean": 0.7599999904632568, |
| "rewards/correctness/std": 0.43141913414001465, |
| "rewards/length_penalty/mean": -0.7114551067352295, |
| "rewards/length_penalty/std": 0.21497680246829987, |
| "sampling/importance_sampling_ratio/max": 1.7750171422958374, |
| "sampling/importance_sampling_ratio/mean": 0.9901482462882996, |
| "sampling/importance_sampling_ratio/min": 0.3599497675895691, |
| "sampling/sampling_logp_difference/max": 1.021790862083435, |
| "sampling/sampling_logp_difference/mean": 0.018604418262839317, |
| "step": 25, |
| "step_time": 25.64425123319961 |
| }, |
| { |
| "clip_ratio/high_max": 0.000790244567906484, |
| "clip_ratio/high_mean": 0.000790244567906484, |
| "clip_ratio/low_mean": 4.310927906772122e-05, |
| "clip_ratio/low_min": 4.310927906772122e-05, |
| "clip_ratio/region_mean": 0.0008333538484293967, |
| "completions/clipped_ratio": 0.07999999821186066, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2031.0, |
| "completions/mean_length": 1346.1199951171875, |
| "completions/mean_terminated_length": 1285.0870361328125, |
| "completions/min_length": 729.0, |
| "completions/min_terminated_length": 729.0, |
| "entropy": 0.2296551436185837, |
| "epoch": 0.07065217391304347, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.020984938368201256, |
| "learning_rate": 5e-05, |
| "loss": 0.09681613743305206, |
| "num_tokens": 2247461.0, |
| "reward": -0.09728515148162842, |
| "reward_std": 0.5121884346008301, |
| "rewards/correctness/mean": 0.5600000023841858, |
| "rewards/correctness/std": 0.5014265179634094, |
| "rewards/length_penalty/mean": -0.6572851538658142, |
| "rewards/length_penalty/std": 0.19092224538326263, |
| "sampling/importance_sampling_ratio/max": 1.9684982299804688, |
| "sampling/importance_sampling_ratio/mean": 0.9919283986091614, |
| "sampling/importance_sampling_ratio/min": 0.34581977128982544, |
| "sampling/sampling_logp_difference/max": 1.0618375539779663, |
| "sampling/sampling_logp_difference/mean": 0.015773361548781395, |
| "step": 26, |
| "step_time": 25.79908507759683 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005374234810005873, |
| "clip_ratio/high_mean": 0.0005374234810005873, |
| "clip_ratio/low_mean": 0.00010813681001309305, |
| "clip_ratio/low_min": 0.00010813681001309305, |
| "clip_ratio/region_mean": 0.0006455602881032974, |
| "completions/clipped_ratio": 0.3999999761581421, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1926.0, |
| "completions/mean_length": 1517.8599853515625, |
| "completions/mean_terminated_length": 1164.433349609375, |
| "completions/min_length": 606.0, |
| "completions/min_terminated_length": 606.0, |
| "entropy": 0.2029076486825943, |
| "epoch": 0.07336956521739131, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017958836629986763, |
| "learning_rate": 5e-05, |
| "loss": 0.028417475521564484, |
| "num_tokens": 2325604.0, |
| "reward": -0.12114257365465164, |
| "reward_std": 0.709344208240509, |
| "rewards/correctness/mean": 0.6200000047683716, |
| "rewards/correctness/std": 0.4903143644332886, |
| "rewards/length_penalty/mean": -0.7411425709724426, |
| "rewards/length_penalty/std": 0.2490270882844925, |
| "sampling/importance_sampling_ratio/max": 1.7574549913406372, |
| "sampling/importance_sampling_ratio/mean": 0.9929352402687073, |
| "sampling/importance_sampling_ratio/min": 0.2425677329301834, |
| "sampling/sampling_logp_difference/max": 1.4164743423461914, |
| "sampling/sampling_logp_difference/mean": 0.013834730722010136, |
| "step": 27, |
| "step_time": 25.780867578927428 |
| }, |
| { |
| "clip_ratio/high_max": 0.0008416285156272352, |
| "clip_ratio/high_mean": 0.0008416285156272352, |
| "clip_ratio/low_mean": 4.83594267279841e-05, |
| "clip_ratio/low_min": 4.83594267279841e-05, |
| "clip_ratio/region_mean": 0.0008899879525415599, |
| "completions/clipped_ratio": 0.05999999865889549, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1903.0, |
| "completions/mean_length": 1254.239990234375, |
| "completions/mean_terminated_length": 1203.574462890625, |
| "completions/min_length": 796.0, |
| "completions/min_terminated_length": 796.0, |
| "entropy": 0.18921910226345062, |
| "epoch": 0.07608695652173914, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.019402744248509407, |
| "learning_rate": 5e-05, |
| "loss": 0.108873650431633, |
| "num_tokens": 2391676.0, |
| "reward": 0.12757812440395355, |
| "reward_std": 0.5219429731369019, |
| "rewards/correctness/mean": 0.7400000095367432, |
| "rewards/correctness/std": 0.44308748841285706, |
| "rewards/length_penalty/mean": -0.6124218702316284, |
| "rewards/length_penalty/std": 0.16508276760578156, |
| "sampling/importance_sampling_ratio/max": 1.7898136377334595, |
| "sampling/importance_sampling_ratio/mean": 0.9934007525444031, |
| "sampling/importance_sampling_ratio/min": 0.2999454140663147, |
| "sampling/sampling_logp_difference/max": 1.2041547298431396, |
| "sampling/sampling_logp_difference/mean": 0.013383659534156322, |
| "step": 28, |
| "step_time": 25.343337242724374 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007759740750771016, |
| "clip_ratio/high_mean": 0.0007759740750771016, |
| "clip_ratio/low_mean": 8.760135315242224e-05, |
| "clip_ratio/low_min": 8.760135315242224e-05, |
| "clip_ratio/region_mean": 0.0008635754231363535, |
| "completions/clipped_ratio": 0.3799999952316284, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2005.0, |
| "completions/mean_length": 1616.659912109375, |
| "completions/mean_terminated_length": 1352.290283203125, |
| "completions/min_length": 575.0, |
| "completions/min_terminated_length": 575.0, |
| "entropy": 0.24235570728778838, |
| "epoch": 0.07880434782608696, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.019807059317827225, |
| "learning_rate": 5e-05, |
| "loss": 0.06528324633836746, |
| "num_tokens": 2475949.0, |
| "reward": -0.12938475608825684, |
| "reward_std": 0.6638873219490051, |
| "rewards/correctness/mean": 0.6600000262260437, |
| "rewards/correctness/std": 0.47851812839508057, |
| "rewards/length_penalty/mean": -0.7893847823143005, |
| "rewards/length_penalty/std": 0.25617480278015137, |
| "sampling/importance_sampling_ratio/max": 1.7938244342803955, |
| "sampling/importance_sampling_ratio/mean": 0.9914634823799133, |
| "sampling/importance_sampling_ratio/min": 0.3581945598125458, |
| "sampling/sampling_logp_difference/max": 1.0266790390014648, |
| "sampling/sampling_logp_difference/mean": 0.01648593135178089, |
| "step": 29, |
| "step_time": 26.831963051110506 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007342056138440967, |
| "clip_ratio/high_mean": 0.0007342056138440967, |
| "clip_ratio/low_mean": 6.386294553522021e-05, |
| "clip_ratio/low_min": 6.386294553522021e-05, |
| "clip_ratio/region_mean": 0.000798068568110466, |
| "completions/clipped_ratio": 0.09999999403953552, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1963.0, |
| "completions/mean_length": 1230.659912109375, |
| "completions/mean_terminated_length": 1139.844482421875, |
| "completions/min_length": 761.0, |
| "completions/min_terminated_length": 761.0, |
| "entropy": 0.1697321891784668, |
| "epoch": 0.08152173913043478, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.015965549275279045, |
| "learning_rate": 5e-05, |
| "loss": 0.07354339957237244, |
| "num_tokens": 2539272.0, |
| "reward": 0.319091796875, |
| "reward_std": 0.4232766628265381, |
| "rewards/correctness/mean": 0.9200000166893005, |
| "rewards/correctness/std": 0.27404749393463135, |
| "rewards/length_penalty/mean": -0.6009082198143005, |
| "rewards/length_penalty/std": 0.19723902642726898, |
| "sampling/importance_sampling_ratio/max": 1.9091259241104126, |
| "sampling/importance_sampling_ratio/mean": 0.9938931465148926, |
| "sampling/importance_sampling_ratio/min": 0.006599857937544584, |
| "sampling/sampling_logp_difference/max": 5.020707130432129, |
| "sampling/sampling_logp_difference/mean": 0.012903784401714802, |
| "step": 30, |
| "step_time": 24.605497697601095 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005373104766476899, |
| "clip_ratio/high_mean": 0.0005373104766476899, |
| "clip_ratio/low_mean": 0.00013891156122554095, |
| "clip_ratio/low_min": 0.00013891156122554095, |
| "clip_ratio/region_mean": 0.0006762220233213157, |
| "completions/clipped_ratio": 0.3400000035762787, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1972.0, |
| "completions/mean_length": 1636.5999755859375, |
| "completions/mean_terminated_length": 1424.666748046875, |
| "completions/min_length": 671.0, |
| "completions/min_terminated_length": 671.0, |
| "entropy": 0.18908907771110534, |
| "epoch": 0.08423913043478261, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01748133823275566, |
| "learning_rate": 5e-05, |
| "loss": 0.05618336424231529, |
| "num_tokens": 2624232.0, |
| "reward": -0.4791210889816284, |
| "reward_std": 0.46296438574790955, |
| "rewards/correctness/mean": 0.3199999928474426, |
| "rewards/correctness/std": 0.4712120592594147, |
| "rewards/length_penalty/mean": -0.799121081829071, |
| "rewards/length_penalty/std": 0.2133444994688034, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.993071436882019, |
| "sampling/importance_sampling_ratio/min": 0.3394291400909424, |
| "sampling/sampling_logp_difference/max": 1.2877426147460938, |
| "sampling/sampling_logp_difference/mean": 0.013400268740952015, |
| "step": 31, |
| "step_time": 27.240520113147795 |
| }, |
| { |
| "clip_ratio/high_max": 0.0011355961789377034, |
| "clip_ratio/high_mean": 0.0011355961789377034, |
| "clip_ratio/low_mean": 0.00015650656423531472, |
| "clip_ratio/low_min": 0.00015650656423531472, |
| "clip_ratio/region_mean": 0.0012921027606353163, |
| "completions/clipped_ratio": 0.11999999731779099, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2022.0, |
| "completions/mean_length": 1262.5599365234375, |
| "completions/mean_terminated_length": 1155.45458984375, |
| "completions/min_length": 595.0, |
| "completions/min_terminated_length": 595.0, |
| "entropy": 0.22945527732372284, |
| "epoch": 0.08695652173913043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01773648150265217, |
| "learning_rate": 5e-05, |
| "loss": 0.09414535760879517, |
| "num_tokens": 2689290.0, |
| "reward": 0.2235156148672104, |
| "reward_std": 0.5308910012245178, |
| "rewards/correctness/mean": 0.8399999737739563, |
| "rewards/correctness/std": 0.37032803893089294, |
| "rewards/length_penalty/mean": -0.6164844036102295, |
| "rewards/length_penalty/std": 0.2093491554260254, |
| "sampling/importance_sampling_ratio/max": 2.3375766277313232, |
| "sampling/importance_sampling_ratio/mean": 0.9917176961898804, |
| "sampling/importance_sampling_ratio/min": 0.30112117528915405, |
| "sampling/sampling_logp_difference/max": 1.200242519378662, |
| "sampling/sampling_logp_difference/mean": 0.01688900962471962, |
| "step": 32, |
| "step_time": 24.735927454195917 |
| }, |
| { |
| "clip_ratio/high_max": 0.000496267294511199, |
| "clip_ratio/high_mean": 0.000496267294511199, |
| "clip_ratio/low_mean": 0.00010696919343899935, |
| "clip_ratio/low_min": 0.00010696919343899935, |
| "clip_ratio/region_mean": 0.0006032364792190493, |
| "completions/clipped_ratio": 0.25999999046325684, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1985.0, |
| "completions/mean_length": 1416.780029296875, |
| "completions/mean_terminated_length": 1195.0, |
| "completions/min_length": 765.0, |
| "completions/min_terminated_length": 765.0, |
| "entropy": 0.19317569732666015, |
| "epoch": 0.08967391304347826, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01730223558843136, |
| "learning_rate": 5e-05, |
| "loss": 0.029047608375549316, |
| "num_tokens": 2763769.0, |
| "reward": -0.09178710728883743, |
| "reward_std": 0.6523070335388184, |
| "rewards/correctness/mean": 0.6000000238418579, |
| "rewards/correctness/std": 0.4948716163635254, |
| "rewards/length_penalty/mean": -0.6917871236801147, |
| "rewards/length_penalty/std": 0.2441810518503189, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9930633902549744, |
| "sampling/importance_sampling_ratio/min": 0.23537996411323547, |
| "sampling/sampling_logp_difference/max": 1.446554183959961, |
| "sampling/sampling_logp_difference/mean": 0.014113553799688816, |
| "step": 33, |
| "step_time": 26.25649970700033 |
| }, |
| { |
| "clip_ratio/high_max": 0.0010051162447780372, |
| "clip_ratio/high_mean": 0.0010051162447780372, |
| "clip_ratio/low_mean": 7.109031575964763e-05, |
| "clip_ratio/low_min": 7.109031575964763e-05, |
| "clip_ratio/region_mean": 0.0010762065765447914, |
| "completions/clipped_ratio": 0.29999998211860657, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1776.0, |
| "completions/mean_length": 1481.6400146484375, |
| "completions/mean_terminated_length": 1238.914306640625, |
| "completions/min_length": 812.0, |
| "completions/min_terminated_length": 812.0, |
| "entropy": 0.23732842803001403, |
| "epoch": 0.09239130434782608, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.020243240520358086, |
| "learning_rate": 5e-05, |
| "loss": 0.11132572591304779, |
| "num_tokens": 2841741.0, |
| "reward": -0.04345703125, |
| "reward_std": 0.6613745093345642, |
| "rewards/correctness/mean": 0.6800000071525574, |
| "rewards/correctness/std": 0.4712120592594147, |
| "rewards/length_penalty/mean": -0.7234570384025574, |
| "rewards/length_penalty/std": 0.21751287579536438, |
| "sampling/importance_sampling_ratio/max": 2.625032424926758, |
| "sampling/importance_sampling_ratio/mean": 0.9916952252388, |
| "sampling/importance_sampling_ratio/min": 0.20833513140678406, |
| "sampling/sampling_logp_difference/max": 1.568607211112976, |
| "sampling/sampling_logp_difference/mean": 0.01689102314412594, |
| "step": 34, |
| "step_time": 26.093267522752285 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007707345008384437, |
| "clip_ratio/high_mean": 0.0007707345008384437, |
| "clip_ratio/low_mean": 5.561825819313526e-05, |
| "clip_ratio/low_min": 5.561825819313526e-05, |
| "clip_ratio/region_mean": 0.0008263527532108128, |
| "completions/clipped_ratio": 0.25999999046325684, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1967.0, |
| "completions/mean_length": 1388.7999267578125, |
| "completions/mean_terminated_length": 1157.189208984375, |
| "completions/min_length": 717.0, |
| "completions/min_terminated_length": 717.0, |
| "entropy": 0.25725961923599244, |
| "epoch": 0.09510869565217392, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.020236779004335403, |
| "learning_rate": 5e-05, |
| "loss": 0.10550195723772049, |
| "num_tokens": 2913911.0, |
| "reward": 0.06187499687075615, |
| "reward_std": 0.646725058555603, |
| "rewards/correctness/mean": 0.7400000095367432, |
| "rewards/correctness/std": 0.44308751821517944, |
| "rewards/length_penalty/mean": -0.6781250238418579, |
| "rewards/length_penalty/std": 0.22613263130187988, |
| "sampling/importance_sampling_ratio/max": 2.0787932872772217, |
| "sampling/importance_sampling_ratio/mean": 0.990626335144043, |
| "sampling/importance_sampling_ratio/min": 0.1848619133234024, |
| "sampling/sampling_logp_difference/max": 1.6881461143493652, |
| "sampling/sampling_logp_difference/mean": 0.018169380724430084, |
| "step": 35, |
| "step_time": 25.364839713089168 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006121197395259515, |
| "clip_ratio/high_mean": 0.0006121197395259515, |
| "clip_ratio/low_mean": 0.00011545439338078722, |
| "clip_ratio/low_min": 0.00011545439338078722, |
| "clip_ratio/region_mean": 0.0007275741430930793, |
| "completions/clipped_ratio": 0.11999999731779099, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2018.0, |
| "completions/mean_length": 1189.8399658203125, |
| "completions/mean_terminated_length": 1072.8182373046875, |
| "completions/min_length": 626.0, |
| "completions/min_terminated_length": 626.0, |
| "entropy": 0.20621423423290253, |
| "epoch": 0.09782608695652174, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017101073637604713, |
| "learning_rate": 5e-05, |
| "loss": 0.08683709055185318, |
| "num_tokens": 2975873.0, |
| "reward": 0.2190234363079071, |
| "reward_std": 0.609960675239563, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.5809765458106995, |
| "rewards/length_penalty/std": 0.22153306007385254, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.992645263671875, |
| "sampling/importance_sampling_ratio/min": 0.14949937164783478, |
| "sampling/sampling_logp_difference/max": 1.9004631042480469, |
| "sampling/sampling_logp_difference/mean": 0.01638173498213291, |
| "step": 36, |
| "step_time": 25.151533517986536 |
| }, |
| { |
| "clip_ratio/high_max": 0.0017051160102710128, |
| "clip_ratio/high_mean": 0.0017051160102710128, |
| "clip_ratio/low_mean": 4.560533707262948e-05, |
| "clip_ratio/low_min": 4.560533707262948e-05, |
| "clip_ratio/region_mean": 0.001750721363350749, |
| "completions/clipped_ratio": 0.23999999463558197, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1696.0, |
| "completions/mean_length": 1309.4599609375, |
| "completions/mean_terminated_length": 1076.23681640625, |
| "completions/min_length": 674.0, |
| "completions/min_terminated_length": 674.0, |
| "entropy": 0.2457641303539276, |
| "epoch": 0.10054347826086957, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01708856225013733, |
| "learning_rate": 5e-05, |
| "loss": 0.08991742134094238, |
| "num_tokens": 3044866.0, |
| "reward": -0.07938476651906967, |
| "reward_std": 0.6378496289253235, |
| "rewards/correctness/mean": 0.5600000023841858, |
| "rewards/correctness/std": 0.5014265179634094, |
| "rewards/length_penalty/mean": -0.6393847465515137, |
| "rewards/length_penalty/std": 0.23324140906333923, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9911713600158691, |
| "sampling/importance_sampling_ratio/min": 0.18068967759609222, |
| "sampling/sampling_logp_difference/max": 1.7109742164611816, |
| "sampling/sampling_logp_difference/mean": 0.01840140111744404, |
| "step": 37, |
| "step_time": 25.540588906034827 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007167195668444038, |
| "clip_ratio/high_mean": 0.0007167195668444038, |
| "clip_ratio/low_mean": 7.254186493810266e-05, |
| "clip_ratio/low_min": 7.254186493810266e-05, |
| "clip_ratio/region_mean": 0.0007892614346928894, |
| "completions/clipped_ratio": 0.11999999731779099, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1988.0, |
| "completions/mean_length": 1333.4000244140625, |
| "completions/mean_terminated_length": 1235.95458984375, |
| "completions/min_length": 690.0, |
| "completions/min_terminated_length": 690.0, |
| "entropy": 0.17886880338191985, |
| "epoch": 0.10326086956521739, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02032496966421604, |
| "learning_rate": 5e-05, |
| "loss": 0.08079551160335541, |
| "num_tokens": 3114926.0, |
| "reward": 0.16892577707767487, |
| "reward_std": 0.5259281396865845, |
| "rewards/correctness/mean": 0.8199999928474426, |
| "rewards/correctness/std": 0.3880879282951355, |
| "rewards/length_penalty/mean": -0.651074230670929, |
| "rewards/length_penalty/std": 0.23055696487426758, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9937843680381775, |
| "sampling/importance_sampling_ratio/min": 0.2627275884151459, |
| "sampling/sampling_logp_difference/max": 1.3366376161575317, |
| "sampling/sampling_logp_difference/mean": 0.013650714419782162, |
| "step": 38, |
| "step_time": 26.08059739181772 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005097337998449802, |
| "clip_ratio/high_mean": 0.0005097337998449802, |
| "clip_ratio/low_mean": 6.99934986187145e-05, |
| "clip_ratio/low_min": 6.99934986187145e-05, |
| "clip_ratio/region_mean": 0.0005797273013740778, |
| "completions/clipped_ratio": 0.2199999988079071, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1743.0, |
| "completions/mean_length": 1207.47998046875, |
| "completions/mean_terminated_length": 970.4102783203125, |
| "completions/min_length": 488.0, |
| "completions/min_terminated_length": 488.0, |
| "entropy": 0.18856219053268433, |
| "epoch": 0.10597826086956522, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.020054886117577553, |
| "learning_rate": 5e-05, |
| "loss": 0.0812826082110405, |
| "num_tokens": 3177790.0, |
| "reward": -0.009589843451976776, |
| "reward_std": 0.7438336610794067, |
| "rewards/correctness/mean": 0.5799999833106995, |
| "rewards/correctness/std": 0.49856939911842346, |
| "rewards/length_penalty/mean": -0.5895898342132568, |
| "rewards/length_penalty/std": 0.26728230714797974, |
| "sampling/importance_sampling_ratio/max": 2.8230764865875244, |
| "sampling/importance_sampling_ratio/mean": 0.9934998750686646, |
| "sampling/importance_sampling_ratio/min": 0.23803837597370148, |
| "sampling/sampling_logp_difference/max": 1.4353233575820923, |
| "sampling/sampling_logp_difference/mean": 0.01414400339126587, |
| "step": 39, |
| "step_time": 24.726267321035266 |
| }, |
| { |
| "clip_ratio/high_max": 0.0008135152223985642, |
| "clip_ratio/high_mean": 0.0008135152223985642, |
| "clip_ratio/low_mean": 8.447097934549674e-05, |
| "clip_ratio/low_min": 8.447097934549674e-05, |
| "clip_ratio/region_mean": 0.0008979861973784864, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1624.0, |
| "completions/max_terminated_length": 1624.0, |
| "completions/mean_length": 933.5399780273438, |
| "completions/mean_terminated_length": 933.5399780273438, |
| "completions/min_length": 641.0, |
| "completions/min_terminated_length": 641.0, |
| "entropy": 0.192103710770607, |
| "epoch": 0.10869565217391304, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.014751175418496132, |
| "learning_rate": 5e-05, |
| "loss": 0.07916796952486038, |
| "num_tokens": 3227187.0, |
| "reward": 0.5441699028015137, |
| "reward_std": 0.11200839281082153, |
| "rewards/correctness/mean": 1.0, |
| "rewards/correctness/std": 0.0, |
| "rewards/length_penalty/mean": -0.45583006739616394, |
| "rewards/length_penalty/std": 0.11200839281082153, |
| "sampling/importance_sampling_ratio/max": 2.518812656402588, |
| "sampling/importance_sampling_ratio/mean": 0.9928264021873474, |
| "sampling/importance_sampling_ratio/min": 0.1412581503391266, |
| "sampling/sampling_logp_difference/max": 1.9571661949157715, |
| "sampling/sampling_logp_difference/mean": 0.016277069225907326, |
| "step": 40, |
| "step_time": 19.264488755259663 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006308750482276082, |
| "clip_ratio/high_mean": 0.0006308750482276082, |
| "clip_ratio/low_mean": 7.04108999343589e-05, |
| "clip_ratio/low_min": 7.04108999343589e-05, |
| "clip_ratio/region_mean": 0.0007012859452515841, |
| "completions/clipped_ratio": 0.17999999225139618, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1934.0, |
| "completions/mean_length": 1309.0999755859375, |
| "completions/mean_terminated_length": 1146.90234375, |
| "completions/min_length": 592.0, |
| "completions/min_terminated_length": 592.0, |
| "entropy": 0.21561425626277925, |
| "epoch": 0.11141304347826086, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.020088374614715576, |
| "learning_rate": 5e-05, |
| "loss": 0.08004692196846008, |
| "num_tokens": 3296382.0, |
| "reward": -0.13920898735523224, |
| "reward_std": 0.6492041945457458, |
| "rewards/correctness/mean": 0.5, |
| "rewards/correctness/std": 0.5050762891769409, |
| "rewards/length_penalty/mean": -0.639208972454071, |
| "rewards/length_penalty/std": 0.229248046875, |
| "sampling/importance_sampling_ratio/max": 2.644029140472412, |
| "sampling/importance_sampling_ratio/mean": 0.9920929670333862, |
| "sampling/importance_sampling_ratio/min": 0.07542523741722107, |
| "sampling/sampling_logp_difference/max": 2.58461332321167, |
| "sampling/sampling_logp_difference/mean": 0.016478460282087326, |
| "step": 41, |
| "step_time": 26.089902371168137 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007611454464495182, |
| "clip_ratio/high_mean": 0.0007611454464495182, |
| "clip_ratio/low_mean": 8.782477234490216e-05, |
| "clip_ratio/low_min": 8.782477234490216e-05, |
| "clip_ratio/region_mean": 0.0008489702187944204, |
| "completions/clipped_ratio": 0.1599999964237213, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1938.0, |
| "completions/mean_length": 1232.4000244140625, |
| "completions/mean_terminated_length": 1077.047607421875, |
| "completions/min_length": 473.0, |
| "completions/min_terminated_length": 473.0, |
| "entropy": 0.19867940843105317, |
| "epoch": 0.11413043478260869, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01986713334918022, |
| "learning_rate": 5e-05, |
| "loss": 0.06985890865325928, |
| "num_tokens": 3360992.0, |
| "reward": 0.23824217915534973, |
| "reward_std": 0.5804083347320557, |
| "rewards/correctness/mean": 0.8399999737739563, |
| "rewards/correctness/std": 0.37032803893089294, |
| "rewards/length_penalty/mean": -0.601757824420929, |
| "rewards/length_penalty/std": 0.2639937698841095, |
| "sampling/importance_sampling_ratio/max": 2.2874951362609863, |
| "sampling/importance_sampling_ratio/mean": 0.9928309917449951, |
| "sampling/importance_sampling_ratio/min": 0.11545022577047348, |
| "sampling/sampling_logp_difference/max": 2.1589157581329346, |
| "sampling/sampling_logp_difference/mean": 0.015996571630239487, |
| "step": 42, |
| "step_time": 24.868821669137105 |
| }, |
| { |
| "clip_ratio/high_max": 0.000910443312022835, |
| "clip_ratio/high_mean": 0.000910443312022835, |
| "clip_ratio/low_mean": 6.609660922549666e-05, |
| "clip_ratio/low_min": 6.609660922549666e-05, |
| "clip_ratio/region_mean": 0.0009765399037860334, |
| "completions/clipped_ratio": 0.019999999552965164, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1646.0, |
| "completions/mean_length": 888.1199951171875, |
| "completions/mean_terminated_length": 864.448974609375, |
| "completions/min_length": 459.0, |
| "completions/min_terminated_length": 459.0, |
| "entropy": 0.21161480844020844, |
| "epoch": 0.11684782608695653, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.020094897598028183, |
| "learning_rate": 5e-05, |
| "loss": 0.06470935791730881, |
| "num_tokens": 3407368.0, |
| "reward": 0.48634764552116394, |
| "reward_std": 0.3447928726673126, |
| "rewards/correctness/mean": 0.9200000166893005, |
| "rewards/correctness/std": 0.27404749393463135, |
| "rewards/length_penalty/mean": -0.4336523413658142, |
| "rewards/length_penalty/std": 0.17656175792217255, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9923710227012634, |
| "sampling/importance_sampling_ratio/min": 0.11419941484928131, |
| "sampling/sampling_logp_difference/max": 2.169809103012085, |
| "sampling/sampling_logp_difference/mean": 0.018651597201824188, |
| "step": 43, |
| "step_time": 22.883227334124967 |
| }, |
| { |
| "clip_ratio/high_max": 0.0008881673566065729, |
| "clip_ratio/high_mean": 0.0008881673566065729, |
| "clip_ratio/low_mean": 0.00012054646213073283, |
| "clip_ratio/low_min": 0.00012054646213073283, |
| "clip_ratio/region_mean": 0.001008713839109987, |
| "completions/clipped_ratio": 0.14000000059604645, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1955.0, |
| "completions/mean_length": 1177.280029296875, |
| "completions/mean_terminated_length": 1035.534912109375, |
| "completions/min_length": 586.0, |
| "completions/min_terminated_length": 586.0, |
| "entropy": 0.2084593027830124, |
| "epoch": 0.11956521739130435, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.019616451114416122, |
| "learning_rate": 5e-05, |
| "loss": 0.12185367941856384, |
| "num_tokens": 3469232.0, |
| "reward": 0.28515625, |
| "reward_std": 0.5443253517150879, |
| "rewards/correctness/mean": 0.8600000143051147, |
| "rewards/correctness/std": 0.3505098223686218, |
| "rewards/length_penalty/mean": -0.5748437643051147, |
| "rewards/length_penalty/std": 0.22794675827026367, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9923920035362244, |
| "sampling/importance_sampling_ratio/min": 0.1075812503695488, |
| "sampling/sampling_logp_difference/max": 2.229508876800537, |
| "sampling/sampling_logp_difference/mean": 0.0175347737967968, |
| "step": 44, |
| "step_time": 25.075815413612872 |
| }, |
| { |
| "clip_ratio/high_max": 0.0009058942960109562, |
| "clip_ratio/high_mean": 0.0009058942960109562, |
| "clip_ratio/low_mean": 7.237063837237657e-05, |
| "clip_ratio/low_min": 7.237063837237657e-05, |
| "clip_ratio/region_mean": 0.0009782649285625666, |
| "completions/clipped_ratio": 0.1599999964237213, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2017.0, |
| "completions/mean_length": 1153.760009765625, |
| "completions/mean_terminated_length": 983.4285888671875, |
| "completions/min_length": 451.0, |
| "completions/min_terminated_length": 451.0, |
| "entropy": 0.23205784261226653, |
| "epoch": 0.12228260869565218, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.021139081567525864, |
| "learning_rate": 5e-05, |
| "loss": 0.06741262972354889, |
| "num_tokens": 3529860.0, |
| "reward": 0.15664061903953552, |
| "reward_std": 0.6854303479194641, |
| "rewards/correctness/mean": 0.7200000286102295, |
| "rewards/correctness/std": 0.4535573720932007, |
| "rewards/length_penalty/mean": -0.5633593797683716, |
| "rewards/length_penalty/std": 0.2703804671764374, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9912254214286804, |
| "sampling/importance_sampling_ratio/min": 0.08344912528991699, |
| "sampling/sampling_logp_difference/max": 2.483518123626709, |
| "sampling/sampling_logp_difference/mean": 0.01880672574043274, |
| "step": 45, |
| "step_time": 24.49455920001492 |
| }, |
| { |
| "clip_ratio/high_max": 0.0015542090637609362, |
| "clip_ratio/high_mean": 0.0015542090637609362, |
| "clip_ratio/low_mean": 8.50872034789063e-05, |
| "clip_ratio/low_min": 8.50872034789063e-05, |
| "clip_ratio/region_mean": 0.0016392962774261833, |
| "completions/clipped_ratio": 0.19999998807907104, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1445.0, |
| "completions/mean_length": 1179.5999755859375, |
| "completions/mean_terminated_length": 962.5, |
| "completions/min_length": 551.0, |
| "completions/min_terminated_length": 551.0, |
| "entropy": 0.23213451504707336, |
| "epoch": 0.125, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.016054287552833557, |
| "learning_rate": 5e-05, |
| "loss": 0.054812848567962646, |
| "num_tokens": 3592350.0, |
| "reward": 0.22402343153953552, |
| "reward_std": 0.6261232495307922, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.575976550579071, |
| "rewards/length_penalty/std": 0.23599623143672943, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.991656482219696, |
| "sampling/importance_sampling_ratio/min": 0.0905810073018074, |
| "sampling/sampling_logp_difference/max": 2.401510715484619, |
| "sampling/sampling_logp_difference/mean": 0.018099654465913773, |
| "step": 46, |
| "step_time": 24.68149675289169 |
| }, |
| { |
| "clip_ratio/high_max": 0.0008501063100993633, |
| "clip_ratio/high_mean": 0.0008501063100993633, |
| "clip_ratio/low_mean": 0.00010414896241854877, |
| "clip_ratio/low_min": 0.00010414896241854877, |
| "clip_ratio/region_mean": 0.0009542552754282952, |
| "completions/clipped_ratio": 0.03999999910593033, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1882.0, |
| "completions/mean_length": 944.0199584960938, |
| "completions/mean_terminated_length": 898.0208740234375, |
| "completions/min_length": 576.0, |
| "completions/min_terminated_length": 576.0, |
| "entropy": 0.19239770770072936, |
| "epoch": 0.12771739130434784, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.020118683576583862, |
| "learning_rate": 5e-05, |
| "loss": 0.035656265914440155, |
| "num_tokens": 3641941.0, |
| "reward": 0.4390527307987213, |
| "reward_std": 0.44222262501716614, |
| "rewards/correctness/mean": 0.8999999761581421, |
| "rewards/correctness/std": 0.30304577946662903, |
| "rewards/length_penalty/mean": -0.46094727516174316, |
| "rewards/length_penalty/std": 0.1884309947490692, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9925170540809631, |
| "sampling/importance_sampling_ratio/min": 0.10729321092367172, |
| "sampling/sampling_logp_difference/max": 2.232189893722534, |
| "sampling/sampling_logp_difference/mean": 0.01707930490374565, |
| "step": 47, |
| "step_time": 23.979559221770614 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007360532443271949, |
| "clip_ratio/high_mean": 0.0007360532443271949, |
| "clip_ratio/low_mean": 5.149670469108969e-05, |
| "clip_ratio/low_min": 5.149670469108969e-05, |
| "clip_ratio/region_mean": 0.0007875499461079017, |
| "completions/clipped_ratio": 0.09999999403953552, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2012.0, |
| "completions/mean_length": 1140.6199951171875, |
| "completions/mean_terminated_length": 1039.800048828125, |
| "completions/min_length": 514.0, |
| "completions/min_terminated_length": 514.0, |
| "entropy": 0.18729614913463594, |
| "epoch": 0.13043478260869565, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.019802836701273918, |
| "learning_rate": 5e-05, |
| "loss": 0.06913310289382935, |
| "num_tokens": 3701932.0, |
| "reward": 0.12305663526058197, |
| "reward_std": 0.6152191162109375, |
| "rewards/correctness/mean": 0.6800000071525574, |
| "rewards/correctness/std": 0.47121208906173706, |
| "rewards/length_penalty/mean": -0.5569433569908142, |
| "rewards/length_penalty/std": 0.24529792368412018, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9927173256874084, |
| "sampling/importance_sampling_ratio/min": 0.13456226885318756, |
| "sampling/sampling_logp_difference/max": 2.005728244781494, |
| "sampling/sampling_logp_difference/mean": 0.016090555116534233, |
| "step": 48, |
| "step_time": 24.360856070183218 |
| }, |
| { |
| "clip_ratio/high_max": 0.0014991959935287014, |
| "clip_ratio/high_mean": 0.0014991959935287014, |
| "clip_ratio/low_mean": 5.690939142368734e-05, |
| "clip_ratio/low_min": 5.690939142368734e-05, |
| "clip_ratio/region_mean": 0.001556105344207026, |
| "completions/clipped_ratio": 0.19999998807907104, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1192.0, |
| "completions/mean_length": 1090.3800048828125, |
| "completions/mean_terminated_length": 850.9750366210938, |
| "completions/min_length": 518.0, |
| "completions/min_terminated_length": 518.0, |
| "entropy": 0.23595961928367615, |
| "epoch": 0.1331521739130435, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.015036417171359062, |
| "learning_rate": 5e-05, |
| "loss": 0.054190024733543396, |
| "num_tokens": 3759141.0, |
| "reward": 0.2675878703594208, |
| "reward_std": 0.6470195651054382, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.40406104922294617, |
| "rewards/length_penalty/mean": -0.5324121117591858, |
| "rewards/length_penalty/std": 0.2540016770362854, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9912841320037842, |
| "sampling/importance_sampling_ratio/min": 0.12618088722229004, |
| "sampling/sampling_logp_difference/max": 2.0700387954711914, |
| "sampling/sampling_logp_difference/mean": 0.019528958946466446, |
| "step": 49, |
| "step_time": 24.133017131127417 |
| }, |
| { |
| "clip_ratio/high_max": 0.0015399554860778153, |
| "clip_ratio/high_mean": 0.0015399554860778153, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0015399554860778153, |
| "completions/clipped_ratio": 0.19999998807907104, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1004.0, |
| "completions/mean_length": 981.9400024414062, |
| "completions/mean_terminated_length": 715.4249877929688, |
| "completions/min_length": 329.0, |
| "completions/min_terminated_length": 329.0, |
| "entropy": 0.2847625881433487, |
| "epoch": 0.1358695652173913, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01285274513065815, |
| "learning_rate": 5e-05, |
| "loss": 0.03275208920240402, |
| "num_tokens": 3810648.0, |
| "reward": 0.12053710967302322, |
| "reward_std": 0.7175264954566956, |
| "rewards/correctness/mean": 0.6000000238418579, |
| "rewards/correctness/std": 0.4948716461658478, |
| "rewards/length_penalty/mean": -0.4794628918170929, |
| "rewards/length_penalty/std": 0.27745550870895386, |
| "sampling/importance_sampling_ratio/max": 2.4426708221435547, |
| "sampling/importance_sampling_ratio/mean": 0.9890289306640625, |
| "sampling/importance_sampling_ratio/min": 0.10526978224515915, |
| "sampling/sampling_logp_difference/max": 2.2512288093566895, |
| "sampling/sampling_logp_difference/mean": 0.022413378581404686, |
| "step": 50, |
| "step_time": 24.217485108412802 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006860518013127148, |
| "clip_ratio/high_mean": 0.0006860518013127148, |
| "clip_ratio/low_mean": 6.418603006750346e-05, |
| "clip_ratio/low_min": 6.418603006750346e-05, |
| "clip_ratio/region_mean": 0.0007502378197386861, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1852.0, |
| "completions/max_terminated_length": 1852.0, |
| "completions/mean_length": 936.3200073242188, |
| "completions/mean_terminated_length": 936.3200073242188, |
| "completions/min_length": 428.0, |
| "completions/min_terminated_length": 428.0, |
| "entropy": 0.19295797646045684, |
| "epoch": 0.13858695652173914, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.016714785248041153, |
| "learning_rate": 5e-05, |
| "loss": 0.08077684044837952, |
| "num_tokens": 3859974.0, |
| "reward": 0.5428124666213989, |
| "reward_std": 0.17791098356246948, |
| "rewards/correctness/mean": 1.0, |
| "rewards/correctness/std": 0.0, |
| "rewards/length_penalty/mean": -0.4571875035762787, |
| "rewards/length_penalty/std": 0.17791098356246948, |
| "sampling/importance_sampling_ratio/max": 2.891409397125244, |
| "sampling/importance_sampling_ratio/mean": 0.992777943611145, |
| "sampling/importance_sampling_ratio/min": 0.12538689374923706, |
| "sampling/sampling_logp_difference/max": 2.0763511657714844, |
| "sampling/sampling_logp_difference/mean": 0.017098676413297653, |
| "step": 51, |
| "step_time": 21.442696853075176 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007797902682796121, |
| "clip_ratio/high_mean": 0.0007797902682796121, |
| "clip_ratio/low_mean": 4.9364475125912576e-05, |
| "clip_ratio/low_min": 4.9364475125912576e-05, |
| "clip_ratio/region_mean": 0.0008291547477710992, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1391.0, |
| "completions/max_terminated_length": 1391.0, |
| "completions/mean_length": 838.8800048828125, |
| "completions/mean_terminated_length": 838.8800048828125, |
| "completions/min_length": 474.0, |
| "completions/min_terminated_length": 474.0, |
| "entropy": 0.17070605158805846, |
| "epoch": 0.14130434782608695, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.015239793807268143, |
| "learning_rate": 5e-05, |
| "loss": 0.03420329466462135, |
| "num_tokens": 3904178.0, |
| "reward": 0.3903906047344208, |
| "reward_std": 0.41191530227661133, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.4096093773841858, |
| "rewards/length_penalty/std": 0.11314770579338074, |
| "sampling/importance_sampling_ratio/max": 2.1938090324401855, |
| "sampling/importance_sampling_ratio/mean": 0.9938638210296631, |
| "sampling/importance_sampling_ratio/min": 0.08321992307901382, |
| "sampling/sampling_logp_difference/max": 2.4862685203552246, |
| "sampling/sampling_logp_difference/mean": 0.01698494143784046, |
| "step": 52, |
| "step_time": 16.543803959153593 |
| }, |
| { |
| "clip_ratio/high_max": 0.0008488356252200902, |
| "clip_ratio/high_mean": 0.0008488356252200902, |
| "clip_ratio/low_mean": 4.159114178037271e-05, |
| "clip_ratio/low_min": 4.159114178037271e-05, |
| "clip_ratio/region_mean": 0.0008904267684556544, |
| "completions/clipped_ratio": 0.07999999821186066, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1766.0, |
| "completions/mean_length": 1033.179931640625, |
| "completions/mean_terminated_length": 944.934814453125, |
| "completions/min_length": 453.0, |
| "completions/min_terminated_length": 453.0, |
| "entropy": 0.20474017262458802, |
| "epoch": 0.14402173913043478, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0204797200858593, |
| "learning_rate": 5e-05, |
| "loss": 0.05927962809801102, |
| "num_tokens": 3958557.0, |
| "reward": 0.19551756978034973, |
| "reward_std": 0.6273199915885925, |
| "rewards/correctness/mean": 0.699999988079071, |
| "rewards/correctness/std": 0.4629100263118744, |
| "rewards/length_penalty/mean": -0.5044824481010437, |
| "rewards/length_penalty/std": 0.22523540258407593, |
| "sampling/importance_sampling_ratio/max": 1.9548008441925049, |
| "sampling/importance_sampling_ratio/mean": 0.991944432258606, |
| "sampling/importance_sampling_ratio/min": 0.04066736251115799, |
| "sampling/sampling_logp_difference/max": 3.202329397201538, |
| "sampling/sampling_logp_difference/mean": 0.0187305249273777, |
| "step": 53, |
| "step_time": 23.953327698865905 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005767365772044286, |
| "clip_ratio/high_mean": 0.0005767365772044286, |
| "clip_ratio/low_mean": 6.257046188693494e-05, |
| "clip_ratio/low_min": 6.257046188693494e-05, |
| "clip_ratio/region_mean": 0.0006393070274498313, |
| "completions/clipped_ratio": 0.19999998807907104, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1871.0, |
| "completions/mean_length": 1254.1199951171875, |
| "completions/mean_terminated_length": 1055.6500244140625, |
| "completions/min_length": 588.0, |
| "completions/min_terminated_length": 588.0, |
| "entropy": 0.20505343973636628, |
| "epoch": 0.14673913043478262, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.018724745139479637, |
| "learning_rate": 5e-05, |
| "loss": 0.07014630734920502, |
| "num_tokens": 4025083.0, |
| "reward": 0.14763671159744263, |
| "reward_std": 0.6248001456260681, |
| "rewards/correctness/mean": 0.7599999904632568, |
| "rewards/correctness/std": 0.43141913414001465, |
| "rewards/length_penalty/mean": -0.6123632788658142, |
| "rewards/length_penalty/std": 0.24844162166118622, |
| "sampling/importance_sampling_ratio/max": 2.6337807178497314, |
| "sampling/importance_sampling_ratio/mean": 0.992132842540741, |
| "sampling/importance_sampling_ratio/min": 0.11581636965274811, |
| "sampling/sampling_logp_difference/max": 2.1557493209838867, |
| "sampling/sampling_logp_difference/mean": 0.01712055131793022, |
| "step": 54, |
| "step_time": 25.70337758003734 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005223329411819577, |
| "clip_ratio/high_mean": 0.0005223329411819577, |
| "clip_ratio/low_mean": 0.00012916228006361053, |
| "clip_ratio/low_min": 0.00012916228006361053, |
| "clip_ratio/region_mean": 0.0006514952285215259, |
| "completions/clipped_ratio": 0.29999998211860657, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1897.0, |
| "completions/mean_length": 1253.8399658203125, |
| "completions/mean_terminated_length": 913.4857177734375, |
| "completions/min_length": 545.0, |
| "completions/min_terminated_length": 545.0, |
| "entropy": 0.24083685576915742, |
| "epoch": 0.14945652173913043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02363269031047821, |
| "learning_rate": 5e-05, |
| "loss": 0.03444070741534233, |
| "num_tokens": 4090665.0, |
| "reward": 0.08777343481779099, |
| "reward_std": 0.7344273328781128, |
| "rewards/correctness/mean": 0.699999988079071, |
| "rewards/correctness/std": 0.4629100263118744, |
| "rewards/length_penalty/mean": -0.6122265458106995, |
| "rewards/length_penalty/std": 0.303725928068161, |
| "sampling/importance_sampling_ratio/max": 2.8206093311309814, |
| "sampling/importance_sampling_ratio/mean": 0.9912818670272827, |
| "sampling/importance_sampling_ratio/min": 0.14987419545650482, |
| "sampling/sampling_logp_difference/max": 1.8979589939117432, |
| "sampling/sampling_logp_difference/mean": 0.019693711772561073, |
| "step": 55, |
| "step_time": 25.123637669952586 |
| }, |
| { |
| "clip_ratio/high_max": 0.0012042871094308794, |
| "clip_ratio/high_mean": 0.0012042871094308794, |
| "clip_ratio/low_mean": 0.00010765771730802953, |
| "clip_ratio/low_min": 0.00010765771730802953, |
| "clip_ratio/region_mean": 0.0013119448209181427, |
| "completions/clipped_ratio": 0.19999998807907104, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1432.0, |
| "completions/mean_length": 987.5599975585938, |
| "completions/mean_terminated_length": 722.4500122070312, |
| "completions/min_length": 398.0, |
| "completions/min_terminated_length": 398.0, |
| "entropy": 0.1985310882329941, |
| "epoch": 0.15217391304347827, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.014590896666049957, |
| "learning_rate": 5e-05, |
| "loss": 0.029594430699944496, |
| "num_tokens": 4143113.0, |
| "reward": 0.11779296398162842, |
| "reward_std": 0.673450767993927, |
| "rewards/correctness/mean": 0.6000000238418579, |
| "rewards/correctness/std": 0.4948716461658478, |
| "rewards/length_penalty/mean": -0.4822070300579071, |
| "rewards/length_penalty/std": 0.29239600896835327, |
| "sampling/importance_sampling_ratio/max": 2.725640296936035, |
| "sampling/importance_sampling_ratio/mean": 0.9925079345703125, |
| "sampling/importance_sampling_ratio/min": 0.15510861575603485, |
| "sampling/sampling_logp_difference/max": 1.863629698753357, |
| "sampling/sampling_logp_difference/mean": 0.017994750291109085, |
| "step": 56, |
| "step_time": 24.02179194591008 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005607875806163065, |
| "clip_ratio/high_mean": 0.0005607875806163065, |
| "clip_ratio/low_mean": 0.0001157178936409764, |
| "clip_ratio/low_min": 0.0001157178936409764, |
| "clip_ratio/region_mean": 0.0006765054655261338, |
| "completions/clipped_ratio": 0.19999998807907104, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1959.0, |
| "completions/mean_length": 1183.419921875, |
| "completions/mean_terminated_length": 967.2750244140625, |
| "completions/min_length": 448.0, |
| "completions/min_terminated_length": 448.0, |
| "entropy": 0.24734485745429993, |
| "epoch": 0.15489130434782608, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01673017628490925, |
| "learning_rate": 5e-05, |
| "loss": 0.04797259345650673, |
| "num_tokens": 4205524.0, |
| "reward": 0.22215819358825684, |
| "reward_std": 0.6535060405731201, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.5778418183326721, |
| "rewards/length_penalty/std": 0.30248209834098816, |
| "sampling/importance_sampling_ratio/max": 2.1797726154327393, |
| "sampling/importance_sampling_ratio/mean": 0.9906326532363892, |
| "sampling/importance_sampling_ratio/min": 0.17795737087726593, |
| "sampling/sampling_logp_difference/max": 1.7262113094329834, |
| "sampling/sampling_logp_difference/mean": 0.019809162244200706, |
| "step": 57, |
| "step_time": 25.37448728713207 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006756895920261741, |
| "clip_ratio/high_mean": 0.0006756895920261741, |
| "clip_ratio/low_mean": 0.0001676665517152287, |
| "clip_ratio/low_min": 0.0001676665517152287, |
| "clip_ratio/region_mean": 0.0008433561422862112, |
| "completions/clipped_ratio": 0.23999999463558197, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1995.0, |
| "completions/mean_length": 1314.719970703125, |
| "completions/mean_terminated_length": 1083.157958984375, |
| "completions/min_length": 486.0, |
| "completions/min_terminated_length": 486.0, |
| "entropy": 0.24679793417453766, |
| "epoch": 0.15760869565217392, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.023087836802005768, |
| "learning_rate": 5e-05, |
| "loss": 0.07203301787376404, |
| "num_tokens": 4275070.0, |
| "reward": -0.001953125, |
| "reward_std": 0.7374632358551025, |
| "rewards/correctness/mean": 0.6399999856948853, |
| "rewards/correctness/std": 0.4848732054233551, |
| "rewards/length_penalty/mean": -0.6419531106948853, |
| "rewards/length_penalty/std": 0.2896905243396759, |
| "sampling/importance_sampling_ratio/max": 2.4132602214813232, |
| "sampling/importance_sampling_ratio/mean": 0.9908359050750732, |
| "sampling/importance_sampling_ratio/min": 0.09266611188650131, |
| "sampling/sampling_logp_difference/max": 2.3787524700164795, |
| "sampling/sampling_logp_difference/mean": 0.020349619910120964, |
| "step": 58, |
| "step_time": 25.843243841081858 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007752020144835114, |
| "clip_ratio/high_mean": 0.0007752020144835114, |
| "clip_ratio/low_mean": 0.0001578759402036667, |
| "clip_ratio/low_min": 0.0001578759402036667, |
| "clip_ratio/region_mean": 0.000933077948866412, |
| "completions/clipped_ratio": 0.07999999821186066, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1734.0, |
| "completions/mean_length": 909.1599731445312, |
| "completions/mean_terminated_length": 810.1304321289062, |
| "completions/min_length": 438.0, |
| "completions/min_terminated_length": 438.0, |
| "entropy": 0.1871820569038391, |
| "epoch": 0.16032608695652173, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.021188728511333466, |
| "learning_rate": 5e-05, |
| "loss": 0.07215945422649384, |
| "num_tokens": 4323238.0, |
| "reward": 0.4360741972923279, |
| "reward_std": 0.5008804798126221, |
| "rewards/correctness/mean": 0.8799999952316284, |
| "rewards/correctness/std": 0.32826071977615356, |
| "rewards/length_penalty/mean": -0.44392576813697815, |
| "rewards/length_penalty/std": 0.22381433844566345, |
| "sampling/importance_sampling_ratio/max": 2.9011504650115967, |
| "sampling/importance_sampling_ratio/mean": 0.9930024743080139, |
| "sampling/importance_sampling_ratio/min": 0.1250293254852295, |
| "sampling/sampling_logp_difference/max": 2.079206943511963, |
| "sampling/sampling_logp_difference/mean": 0.018969286233186722, |
| "step": 59, |
| "step_time": 23.926262149121612 |
| }, |
| { |
| "clip_ratio/high_max": 0.0004012422810774297, |
| "clip_ratio/high_mean": 0.0004012422810774297, |
| "clip_ratio/low_mean": 4.353504627943039e-05, |
| "clip_ratio/low_min": 4.353504627943039e-05, |
| "clip_ratio/region_mean": 0.00044477732735686003, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1467.0, |
| "completions/max_terminated_length": 1467.0, |
| "completions/mean_length": 893.9400024414062, |
| "completions/mean_terminated_length": 893.9400024414062, |
| "completions/min_length": 389.0, |
| "completions/min_terminated_length": 389.0, |
| "entropy": 0.15323590338230134, |
| "epoch": 0.16304347826086957, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.013462953269481659, |
| "learning_rate": 5e-05, |
| "loss": 0.026647938415408134, |
| "num_tokens": 4370745.0, |
| "reward": -0.05649413913488388, |
| "reward_std": 0.5149266123771667, |
| "rewards/correctness/mean": 0.3799999952316284, |
| "rewards/correctness/std": 0.4903143644332886, |
| "rewards/length_penalty/mean": -0.4364941418170929, |
| "rewards/length_penalty/std": 0.13239894807338715, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9946074485778809, |
| "sampling/importance_sampling_ratio/min": 0.07585878670215607, |
| "sampling/sampling_logp_difference/max": 2.5788817405700684, |
| "sampling/sampling_logp_difference/mean": 0.015621195547282696, |
| "step": 60, |
| "step_time": 17.78750504530035 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007783836219459773, |
| "clip_ratio/high_mean": 0.0007783836219459773, |
| "clip_ratio/low_mean": 7.167461153585463e-05, |
| "clip_ratio/low_min": 7.167461153585463e-05, |
| "clip_ratio/region_mean": 0.0008500582538545131, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1332.0, |
| "completions/max_terminated_length": 1332.0, |
| "completions/mean_length": 816.8800048828125, |
| "completions/mean_terminated_length": 816.8800048828125, |
| "completions/min_length": 530.0, |
| "completions/min_terminated_length": 530.0, |
| "entropy": 0.16096674501895905, |
| "epoch": 0.16576086956521738, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01674392633140087, |
| "learning_rate": 5e-05, |
| "loss": 0.04878897964954376, |
| "num_tokens": 4413999.0, |
| "reward": 0.6011328101158142, |
| "reward_std": 0.08173488825559616, |
| "rewards/correctness/mean": 1.0, |
| "rewards/correctness/std": 0.0, |
| "rewards/length_penalty/mean": -0.3988671898841858, |
| "rewards/length_penalty/std": 0.08173488825559616, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9938045740127563, |
| "sampling/importance_sampling_ratio/min": 0.03371766582131386, |
| "sampling/sampling_logp_difference/max": 3.38973331451416, |
| "sampling/sampling_logp_difference/mean": 0.01792747713625431, |
| "step": 61, |
| "step_time": 15.884311718866229 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005490639596246183, |
| "clip_ratio/high_mean": 0.0005490639596246183, |
| "clip_ratio/low_mean": 8.991758077172562e-05, |
| "clip_ratio/low_min": 8.991758077172562e-05, |
| "clip_ratio/region_mean": 0.0006389815418515355, |
| "completions/clipped_ratio": 0.05999999865889549, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1749.0, |
| "completions/mean_length": 1040.5399169921875, |
| "completions/mean_terminated_length": 976.2340087890625, |
| "completions/min_length": 389.0, |
| "completions/min_terminated_length": 389.0, |
| "entropy": 0.15563784837722777, |
| "epoch": 0.16847826086956522, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.018042365089058876, |
| "learning_rate": 5e-05, |
| "loss": 0.07988356798887253, |
| "num_tokens": 4469646.0, |
| "reward": 0.33192381262779236, |
| "reward_std": 0.5288719534873962, |
| "rewards/correctness/mean": 0.8399999737739563, |
| "rewards/correctness/std": 0.37032803893089294, |
| "rewards/length_penalty/mean": -0.5080761909484863, |
| "rewards/length_penalty/std": 0.21403193473815918, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9942488074302673, |
| "sampling/importance_sampling_ratio/min": 0.06006976589560509, |
| "sampling/sampling_logp_difference/max": 2.812248706817627, |
| "sampling/sampling_logp_difference/mean": 0.015519483014941216, |
| "step": 62, |
| "step_time": 24.003339575137943 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007015687064267695, |
| "clip_ratio/high_mean": 0.0007015687064267695, |
| "clip_ratio/low_mean": 0.00010667735186871141, |
| "clip_ratio/low_min": 0.00010667735186871141, |
| "clip_ratio/region_mean": 0.0008082460495643318, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1687.0, |
| "completions/max_terminated_length": 1687.0, |
| "completions/mean_length": 828.7799682617188, |
| "completions/mean_terminated_length": 828.7799682617188, |
| "completions/min_length": 316.0, |
| "completions/min_terminated_length": 316.0, |
| "entropy": 0.16558827161788942, |
| "epoch": 0.17119565217391305, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.016367904841899872, |
| "learning_rate": 5e-05, |
| "loss": 0.04421332851052284, |
| "num_tokens": 4514335.0, |
| "reward": 0.3953222632408142, |
| "reward_std": 0.5528039932250977, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.40467774868011475, |
| "rewards/length_penalty/std": 0.1717558354139328, |
| "sampling/importance_sampling_ratio/max": 2.7936859130859375, |
| "sampling/importance_sampling_ratio/mean": 0.9938439130783081, |
| "sampling/importance_sampling_ratio/min": 0.05281807854771614, |
| "sampling/sampling_logp_difference/max": 2.940901756286621, |
| "sampling/sampling_logp_difference/mean": 0.017519377171993256, |
| "step": 63, |
| "step_time": 20.010241974843666 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006772913970053196, |
| "clip_ratio/high_mean": 0.0006772913970053196, |
| "clip_ratio/low_mean": 6.347343005472795e-05, |
| "clip_ratio/low_min": 6.347343005472795e-05, |
| "clip_ratio/region_mean": 0.0007407648256048561, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 2002.0, |
| "completions/max_terminated_length": 2002.0, |
| "completions/mean_length": 881.5399780273438, |
| "completions/mean_terminated_length": 881.5399780273438, |
| "completions/min_length": 503.0, |
| "completions/min_terminated_length": 503.0, |
| "entropy": 0.16818468570709227, |
| "epoch": 0.17391304347826086, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.016808470711112022, |
| "learning_rate": 5e-05, |
| "loss": 0.010990206152200699, |
| "num_tokens": 4561932.0, |
| "reward": 0.5095605254173279, |
| "reward_std": 0.27923068404197693, |
| "rewards/correctness/mean": 0.9399999976158142, |
| "rewards/correctness/std": 0.2398979365825653, |
| "rewards/length_penalty/mean": -0.43043944239616394, |
| "rewards/length_penalty/std": 0.12210877984762192, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9936347007751465, |
| "sampling/importance_sampling_ratio/min": 0.11113839596509933, |
| "sampling/sampling_logp_difference/max": 2.19697904586792, |
| "sampling/sampling_logp_difference/mean": 0.018134895712137222, |
| "step": 64, |
| "step_time": 22.430961188860238 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006107009830884635, |
| "clip_ratio/high_mean": 0.0006107009830884635, |
| "clip_ratio/low_mean": 0.00012073849065927789, |
| "clip_ratio/low_min": 0.00012073849065927789, |
| "clip_ratio/region_mean": 0.000731439475202933, |
| "completions/clipped_ratio": 0.14000000059604645, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1452.0, |
| "completions/mean_length": 999.5, |
| "completions/mean_terminated_length": 828.81396484375, |
| "completions/min_length": 454.0, |
| "completions/min_terminated_length": 454.0, |
| "entropy": 0.2120996743440628, |
| "epoch": 0.1766304347826087, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01858065277338028, |
| "learning_rate": 5e-05, |
| "loss": 0.07004983723163605, |
| "num_tokens": 4615617.0, |
| "reward": 0.3319628834724426, |
| "reward_std": 0.5958665013313293, |
| "rewards/correctness/mean": 0.8199999928474426, |
| "rewards/correctness/std": 0.3880879282951355, |
| "rewards/length_penalty/mean": -0.488037109375, |
| "rewards/length_penalty/std": 0.233457550406456, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9916940331459045, |
| "sampling/importance_sampling_ratio/min": 0.0842479020357132, |
| "sampling/sampling_logp_difference/max": 2.473991632461548, |
| "sampling/sampling_logp_difference/mean": 0.020551443099975586, |
| "step": 65, |
| "step_time": 23.88459292682819 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006235995824681595, |
| "clip_ratio/high_mean": 0.0006235995824681595, |
| "clip_ratio/low_mean": 0.00022068115940783174, |
| "clip_ratio/low_min": 0.00022068115940783174, |
| "clip_ratio/region_mean": 0.0008442807185929269, |
| "completions/clipped_ratio": 0.14000000059604645, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1759.0, |
| "completions/mean_length": 875.4400024414062, |
| "completions/mean_terminated_length": 684.5581665039062, |
| "completions/min_length": 362.0, |
| "completions/min_terminated_length": 362.0, |
| "entropy": 0.2801995426416397, |
| "epoch": 0.1793478260869565, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01836577244102955, |
| "learning_rate": 5e-05, |
| "loss": 0.0533025860786438, |
| "num_tokens": 4662439.0, |
| "reward": 0.37253904342651367, |
| "reward_std": 0.6679306626319885, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.4274609386920929, |
| "rewards/length_penalty/std": 0.2749612033367157, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9891699552536011, |
| "sampling/importance_sampling_ratio/min": 0.09494689106941223, |
| "sampling/sampling_logp_difference/max": 2.3544375896453857, |
| "sampling/sampling_logp_difference/mean": 0.025045860558748245, |
| "step": 66, |
| "step_time": 24.01824735174887 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006467935570981353, |
| "clip_ratio/high_mean": 0.0006467935570981353, |
| "clip_ratio/low_mean": 4.287245392333716e-05, |
| "clip_ratio/low_min": 4.287245392333716e-05, |
| "clip_ratio/region_mean": 0.0006896660139318556, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1456.0, |
| "completions/max_terminated_length": 1456.0, |
| "completions/mean_length": 827.5399780273438, |
| "completions/mean_terminated_length": 827.5399780273438, |
| "completions/min_length": 275.0, |
| "completions/min_terminated_length": 275.0, |
| "entropy": 0.1334616631269455, |
| "epoch": 0.18206521739130435, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.015385694801807404, |
| "learning_rate": 5e-05, |
| "loss": 0.042910344898700714, |
| "num_tokens": 4706766.0, |
| "reward": 0.1959277242422104, |
| "reward_std": 0.5535297393798828, |
| "rewards/correctness/mean": 0.6000000238418579, |
| "rewards/correctness/std": 0.4948716461658478, |
| "rewards/length_penalty/mean": -0.40407225489616394, |
| "rewards/length_penalty/std": 0.1448303610086441, |
| "sampling/importance_sampling_ratio/max": 2.695427417755127, |
| "sampling/importance_sampling_ratio/mean": 0.9948953986167908, |
| "sampling/importance_sampling_ratio/min": 0.09551125019788742, |
| "sampling/sampling_logp_difference/max": 2.348511219024658, |
| "sampling/sampling_logp_difference/mean": 0.01522344071418047, |
| "step": 67, |
| "step_time": 16.992219537962228 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005427057622000575, |
| "clip_ratio/high_mean": 0.0005427057622000575, |
| "clip_ratio/low_mean": 6.446797924581915e-05, |
| "clip_ratio/low_min": 6.446797924581915e-05, |
| "clip_ratio/region_mean": 0.0006071737268939614, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1989.0, |
| "completions/max_terminated_length": 1989.0, |
| "completions/mean_length": 871.8800048828125, |
| "completions/mean_terminated_length": 871.8800048828125, |
| "completions/min_length": 508.0, |
| "completions/min_terminated_length": 508.0, |
| "entropy": 0.1475384384393692, |
| "epoch": 0.18478260869565216, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017730075865983963, |
| "learning_rate": 5e-05, |
| "loss": 0.05759969353675842, |
| "num_tokens": 4754120.0, |
| "reward": 0.3742773234844208, |
| "reward_std": 0.5240140557289124, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.4257226586341858, |
| "rewards/length_penalty/std": 0.14220459759235382, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9945562481880188, |
| "sampling/importance_sampling_ratio/min": 0.09534671157598495, |
| "sampling/sampling_logp_difference/max": 2.3502354621887207, |
| "sampling/sampling_logp_difference/mean": 0.01613455079495907, |
| "step": 68, |
| "step_time": 22.368666774127632 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006822091600042768, |
| "clip_ratio/high_mean": 0.0006822091600042768, |
| "clip_ratio/low_mean": 0.00017659609293332323, |
| "clip_ratio/low_min": 0.00017659609293332323, |
| "clip_ratio/region_mean": 0.0008588052645791322, |
| "completions/clipped_ratio": 0.1599999964237213, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1823.0, |
| "completions/mean_length": 852.5, |
| "completions/mean_terminated_length": 624.7857055664062, |
| "completions/min_length": 426.0, |
| "completions/min_terminated_length": 426.0, |
| "entropy": 0.22420361936092376, |
| "epoch": 0.1875, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01747017167508602, |
| "learning_rate": 5e-05, |
| "loss": 0.045437250286340714, |
| "num_tokens": 4799125.0, |
| "reward": 0.18374022841453552, |
| "reward_std": 0.6997568011283875, |
| "rewards/correctness/mean": 0.6000000238418579, |
| "rewards/correctness/std": 0.4948716461658478, |
| "rewards/length_penalty/mean": -0.416259765625, |
| "rewards/length_penalty/std": 0.2845315635204315, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.99150550365448, |
| "sampling/importance_sampling_ratio/min": 0.08363517373800278, |
| "sampling/sampling_logp_difference/max": 2.4812910556793213, |
| "sampling/sampling_logp_difference/mean": 0.02138812094926834, |
| "step": 69, |
| "step_time": 23.31697426387109 |
| }, |
| { |
| "clip_ratio/high_max": 0.0008050206408370287, |
| "clip_ratio/high_mean": 0.0008050206408370287, |
| "clip_ratio/low_mean": 5.3399032913148405e-05, |
| "clip_ratio/low_min": 5.3399032913148405e-05, |
| "clip_ratio/region_mean": 0.000858419673750177, |
| "completions/clipped_ratio": 0.19999998807907104, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1751.0, |
| "completions/mean_length": 1082.97998046875, |
| "completions/mean_terminated_length": 841.7250366210938, |
| "completions/min_length": 435.0, |
| "completions/min_terminated_length": 435.0, |
| "entropy": 0.14042216837406157, |
| "epoch": 0.19021739130434784, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01626087725162506, |
| "learning_rate": 5e-05, |
| "loss": 0.04337764531373978, |
| "num_tokens": 4856784.0, |
| "reward": -0.3287988305091858, |
| "reward_std": 0.583001434803009, |
| "rewards/correctness/mean": 0.20000000298023224, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.5287988185882568, |
| "rewards/length_penalty/std": 0.2958765923976898, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9947971701622009, |
| "sampling/importance_sampling_ratio/min": 0.1584167629480362, |
| "sampling/sampling_logp_difference/max": 1.8425259590148926, |
| "sampling/sampling_logp_difference/mean": 0.014615754596889019, |
| "step": 70, |
| "step_time": 24.817883169045672 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006307939882390201, |
| "clip_ratio/high_mean": 0.0006307939882390201, |
| "clip_ratio/low_mean": 0.00010670171468518674, |
| "clip_ratio/low_min": 0.00010670171468518674, |
| "clip_ratio/region_mean": 0.0007374957029242068, |
| "completions/clipped_ratio": 0.03999999910593033, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1123.0, |
| "completions/mean_length": 677.9400024414062, |
| "completions/mean_terminated_length": 620.8541870117188, |
| "completions/min_length": 333.0, |
| "completions/min_terminated_length": 333.0, |
| "entropy": 0.17736737430095673, |
| "epoch": 0.19293478260869565, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.015899593010544777, |
| "learning_rate": 5e-05, |
| "loss": 0.04084380343556404, |
| "num_tokens": 4893041.0, |
| "reward": 0.20897459983825684, |
| "reward_std": 0.5713562965393066, |
| "rewards/correctness/mean": 0.5400000214576721, |
| "rewards/correctness/std": 0.5034574270248413, |
| "rewards/length_penalty/mean": -0.3310253918170929, |
| "rewards/length_penalty/std": 0.16477957367897034, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9929877519607544, |
| "sampling/importance_sampling_ratio/min": 0.0540241040289402, |
| "sampling/sampling_logp_difference/max": 2.9183249473571777, |
| "sampling/sampling_logp_difference/mean": 0.02107720635831356, |
| "step": 71, |
| "step_time": 22.17649055412039 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007707292737904936, |
| "clip_ratio/high_mean": 0.0007707292737904936, |
| "clip_ratio/low_mean": 9.523197077214718e-05, |
| "clip_ratio/low_min": 9.523197077214718e-05, |
| "clip_ratio/region_mean": 0.0008659612271003425, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1150.0, |
| "completions/max_terminated_length": 1150.0, |
| "completions/mean_length": 638.5599975585938, |
| "completions/mean_terminated_length": 638.5599975585938, |
| "completions/min_length": 352.0, |
| "completions/min_terminated_length": 352.0, |
| "entropy": 0.16127082109451293, |
| "epoch": 0.1956521739130435, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.015706723555922508, |
| "learning_rate": 5e-05, |
| "loss": 0.012553978711366653, |
| "num_tokens": 4928639.0, |
| "reward": 0.42820310592651367, |
| "reward_std": 0.5053699612617493, |
| "rewards/correctness/mean": 0.7400000095367432, |
| "rewards/correctness/std": 0.44308751821517944, |
| "rewards/length_penalty/mean": -0.3117968738079071, |
| "rewards/length_penalty/std": 0.0972241684794426, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9939435124397278, |
| "sampling/importance_sampling_ratio/min": 0.07872314006090164, |
| "sampling/sampling_logp_difference/max": 2.541818141937256, |
| "sampling/sampling_logp_difference/mean": 0.020169558003544807, |
| "step": 72, |
| "step_time": 13.558618172770366 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005829828616697342, |
| "clip_ratio/high_mean": 0.0005829828616697342, |
| "clip_ratio/low_mean": 0.00014270264655351638, |
| "clip_ratio/low_min": 0.00014270264655351638, |
| "clip_ratio/region_mean": 0.0007256855198647827, |
| "completions/clipped_ratio": 0.07999999821186066, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1980.0, |
| "completions/mean_length": 888.760009765625, |
| "completions/mean_terminated_length": 787.95654296875, |
| "completions/min_length": 440.0, |
| "completions/min_terminated_length": 440.0, |
| "entropy": 0.19430480599403382, |
| "epoch": 0.1983695652173913, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.023028841242194176, |
| "learning_rate": 5e-05, |
| "loss": 0.03241705894470215, |
| "num_tokens": 4975487.0, |
| "reward": 0.22603514790534973, |
| "reward_std": 0.6428487300872803, |
| "rewards/correctness/mean": 0.6600000262260437, |
| "rewards/correctness/std": 0.47851812839508057, |
| "rewards/length_penalty/mean": -0.4339648485183716, |
| "rewards/length_penalty/std": 0.2522260248661041, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.992754340171814, |
| "sampling/importance_sampling_ratio/min": 0.1667802333831787, |
| "sampling/sampling_logp_difference/max": 1.7910783290863037, |
| "sampling/sampling_logp_difference/mean": 0.019066043198108673, |
| "step": 73, |
| "step_time": 23.85767387226224 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005934039130806923, |
| "clip_ratio/high_mean": 0.0005934039130806923, |
| "clip_ratio/low_mean": 0.00011007156572304667, |
| "clip_ratio/low_min": 0.00011007156572304667, |
| "clip_ratio/region_mean": 0.0007034754729829728, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1745.0, |
| "completions/max_terminated_length": 1745.0, |
| "completions/mean_length": 641.0799560546875, |
| "completions/mean_terminated_length": 641.0799560546875, |
| "completions/min_length": 316.0, |
| "completions/min_terminated_length": 316.0, |
| "entropy": 0.1928534835577011, |
| "epoch": 0.20108695652173914, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.019313158467411995, |
| "learning_rate": 5e-05, |
| "loss": 0.05762927606701851, |
| "num_tokens": 5010871.0, |
| "reward": 0.08697265386581421, |
| "reward_std": 0.5515088438987732, |
| "rewards/correctness/mean": 0.4000000059604645, |
| "rewards/correctness/std": 0.49487167596817017, |
| "rewards/length_penalty/mean": -0.31302735209465027, |
| "rewards/length_penalty/std": 0.12540066242218018, |
| "sampling/importance_sampling_ratio/max": 2.433044910430908, |
| "sampling/importance_sampling_ratio/mean": 0.9928313493728638, |
| "sampling/importance_sampling_ratio/min": 0.08987344801425934, |
| "sampling/sampling_logp_difference/max": 2.4093527793884277, |
| "sampling/sampling_logp_difference/mean": 0.021775977686047554, |
| "step": 74, |
| "step_time": 19.006243550917134 |
| }, |
| { |
| "clip_ratio/high_max": 0.000405387906357646, |
| "clip_ratio/high_mean": 0.000405387906357646, |
| "clip_ratio/low_mean": 0.00012361486005829647, |
| "clip_ratio/low_min": 0.00012361486005829647, |
| "clip_ratio/region_mean": 0.0005290027736919, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1529.0, |
| "completions/max_terminated_length": 1529.0, |
| "completions/mean_length": 633.239990234375, |
| "completions/mean_terminated_length": 633.239990234375, |
| "completions/min_length": 227.0, |
| "completions/min_terminated_length": 227.0, |
| "entropy": 0.17757215201854706, |
| "epoch": 0.20380434782608695, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.020598672330379486, |
| "learning_rate": 5e-05, |
| "loss": 0.03665365278720856, |
| "num_tokens": 5044863.0, |
| "reward": 0.49080076813697815, |
| "reward_std": 0.45810505747795105, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.3091992139816284, |
| "rewards/length_penalty/std": 0.1596784144639969, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9929429292678833, |
| "sampling/importance_sampling_ratio/min": 0.06892305612564087, |
| "sampling/sampling_logp_difference/max": 2.674764633178711, |
| "sampling/sampling_logp_difference/mean": 0.021487612277269363, |
| "step": 75, |
| "step_time": 16.96747936704196 |
| }, |
| { |
| "clip_ratio/high_max": 0.0008531624858733267, |
| "clip_ratio/high_mean": 0.0008531624858733267, |
| "clip_ratio/low_mean": 1.858218020061031e-05, |
| "clip_ratio/low_min": 1.858218020061031e-05, |
| "clip_ratio/region_mean": 0.0008717446587979793, |
| "completions/clipped_ratio": 0.09999999403953552, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1162.0, |
| "completions/mean_length": 658.0599975585938, |
| "completions/mean_terminated_length": 503.6222229003906, |
| "completions/min_length": 172.0, |
| "completions/min_terminated_length": 172.0, |
| "entropy": 0.24603551626205444, |
| "epoch": 0.20652173913043478, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012405374087393284, |
| "learning_rate": 5e-05, |
| "loss": 0.025707000866532326, |
| "num_tokens": 5081106.0, |
| "reward": 0.43868163228034973, |
| "reward_std": 0.6422238349914551, |
| "rewards/correctness/mean": 0.7599999904632568, |
| "rewards/correctness/std": 0.43141910433769226, |
| "rewards/length_penalty/mean": -0.3213183581829071, |
| "rewards/length_penalty/std": 0.255402535200119, |
| "sampling/importance_sampling_ratio/max": 2.560499906539917, |
| "sampling/importance_sampling_ratio/mean": 0.9905421733856201, |
| "sampling/importance_sampling_ratio/min": 0.1971578299999237, |
| "sampling/sampling_logp_difference/max": 1.6237506866455078, |
| "sampling/sampling_logp_difference/mean": 0.0244895052164793, |
| "step": 76, |
| "step_time": 22.57146995374933 |
| }, |
| { |
| "clip_ratio/high_max": 0.0003801504441071302, |
| "clip_ratio/high_mean": 0.0003801504441071302, |
| "clip_ratio/low_mean": 0.00011223704786971212, |
| "clip_ratio/low_min": 0.00011223704786971212, |
| "clip_ratio/region_mean": 0.0004923874977976084, |
| "completions/clipped_ratio": 0.03999999910593033, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1808.0, |
| "completions/mean_length": 628.7799682617188, |
| "completions/mean_terminated_length": 569.6458740234375, |
| "completions/min_length": 214.0, |
| "completions/min_terminated_length": 214.0, |
| "entropy": 0.20442797541618346, |
| "epoch": 0.20923913043478262, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.019539417698979378, |
| "learning_rate": 5e-05, |
| "loss": 0.04475327581167221, |
| "num_tokens": 5115105.0, |
| "reward": 0.552978515625, |
| "reward_std": 0.547501802444458, |
| "rewards/correctness/mean": 0.8600000143051147, |
| "rewards/correctness/std": 0.3505098223686218, |
| "rewards/length_penalty/mean": -0.30702149868011475, |
| "rewards/length_penalty/std": 0.22665424644947052, |
| "sampling/importance_sampling_ratio/max": 2.7733845710754395, |
| "sampling/importance_sampling_ratio/mean": 0.992488443851471, |
| "sampling/importance_sampling_ratio/min": 0.21041618287563324, |
| "sampling/sampling_logp_difference/max": 1.5586678981781006, |
| "sampling/sampling_logp_difference/mean": 0.021170448511838913, |
| "step": 77, |
| "step_time": 22.34465615078807 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006306407682131976, |
| "clip_ratio/high_mean": 0.0006306407682131976, |
| "clip_ratio/low_mean": 3.474635304883123e-05, |
| "clip_ratio/low_min": 3.474635304883123e-05, |
| "clip_ratio/region_mean": 0.0006653871096204966, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1658.0, |
| "completions/max_terminated_length": 1658.0, |
| "completions/mean_length": 599.6599731445312, |
| "completions/mean_terminated_length": 599.6599731445312, |
| "completions/min_length": 300.0, |
| "completions/min_terminated_length": 300.0, |
| "entropy": 0.1244092807173729, |
| "epoch": 0.21195652173913043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.015446798875927925, |
| "learning_rate": 5e-05, |
| "loss": 0.033272288739681244, |
| "num_tokens": 5147388.0, |
| "reward": 0.26719725131988525, |
| "reward_std": 0.5827784538269043, |
| "rewards/correctness/mean": 0.5600000023841858, |
| "rewards/correctness/std": 0.5014265179634094, |
| "rewards/length_penalty/mean": -0.29280272126197815, |
| "rewards/length_penalty/std": 0.13731949031352997, |
| "sampling/importance_sampling_ratio/max": 2.4936909675598145, |
| "sampling/importance_sampling_ratio/mean": 0.994698166847229, |
| "sampling/importance_sampling_ratio/min": 0.31123581528663635, |
| "sampling/sampling_logp_difference/max": 1.1672043800354004, |
| "sampling/sampling_logp_difference/mean": 0.0171463992446661, |
| "step": 78, |
| "step_time": 18.60519856400788 |
| }, |
| { |
| "clip_ratio/high_max": 0.0008912550430977717, |
| "clip_ratio/high_mean": 0.0008912550430977717, |
| "clip_ratio/low_mean": 6.107631779741495e-05, |
| "clip_ratio/low_min": 6.107631779741495e-05, |
| "clip_ratio/region_mean": 0.0009523313667159528, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1229.0, |
| "completions/max_terminated_length": 1229.0, |
| "completions/mean_length": 649.219970703125, |
| "completions/mean_terminated_length": 649.219970703125, |
| "completions/min_length": 359.0, |
| "completions/min_terminated_length": 359.0, |
| "entropy": 0.13126298934221267, |
| "epoch": 0.21467391304347827, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0188016165047884, |
| "learning_rate": 5e-05, |
| "loss": 0.015198985114693642, |
| "num_tokens": 5183309.0, |
| "reward": 0.4429980516433716, |
| "reward_std": 0.4340318739414215, |
| "rewards/correctness/mean": 0.7599999904632568, |
| "rewards/correctness/std": 0.43141913414001465, |
| "rewards/length_penalty/mean": -0.31700193881988525, |
| "rewards/length_penalty/std": 0.08376923203468323, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9953542351722717, |
| "sampling/importance_sampling_ratio/min": 0.24312418699264526, |
| "sampling/sampling_logp_difference/max": 1.5119855403900146, |
| "sampling/sampling_logp_difference/mean": 0.018735092133283615, |
| "step": 79, |
| "step_time": 14.177364142145962 |
| }, |
| { |
| "clip_ratio/high_max": 0.0004742158984299749, |
| "clip_ratio/high_mean": 0.0004742158984299749, |
| "clip_ratio/low_mean": 0.0001656829408602789, |
| "clip_ratio/low_min": 0.0001656829408602789, |
| "clip_ratio/region_mean": 0.0006398988363798708, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1211.0, |
| "completions/max_terminated_length": 1211.0, |
| "completions/mean_length": 603.4199829101562, |
| "completions/mean_terminated_length": 603.4199829101562, |
| "completions/min_length": 246.0, |
| "completions/min_terminated_length": 246.0, |
| "entropy": 0.15212061405181884, |
| "epoch": 0.21739130434782608, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0165855772793293, |
| "learning_rate": 5e-05, |
| "loss": 0.041541777551174164, |
| "num_tokens": 5216640.0, |
| "reward": 0.10536132752895355, |
| "reward_std": 0.5482556223869324, |
| "rewards/correctness/mean": 0.4000000059604645, |
| "rewards/correctness/std": 0.49487167596817017, |
| "rewards/length_penalty/mean": -0.29463866353034973, |
| "rewards/length_penalty/std": 0.07611750066280365, |
| "sampling/importance_sampling_ratio/max": 2.329467535018921, |
| "sampling/importance_sampling_ratio/mean": 0.9940717816352844, |
| "sampling/importance_sampling_ratio/min": 0.20993992686271667, |
| "sampling/sampling_logp_difference/max": 1.5609338283538818, |
| "sampling/sampling_logp_difference/mean": 0.019620422273874283, |
| "step": 80, |
| "step_time": 13.800314102787524 |
| }, |
| { |
| "clip_ratio/high_max": 0.000613589191925712, |
| "clip_ratio/high_mean": 0.000613589191925712, |
| "clip_ratio/low_mean": 8.642562024760992e-05, |
| "clip_ratio/low_min": 8.642562024760992e-05, |
| "clip_ratio/region_mean": 0.0007000148179940879, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1088.0, |
| "completions/max_terminated_length": 1088.0, |
| "completions/mean_length": 649.4199829101562, |
| "completions/mean_terminated_length": 649.4199829101562, |
| "completions/min_length": 292.0, |
| "completions/min_terminated_length": 292.0, |
| "entropy": 0.15520131587982178, |
| "epoch": 0.22010869565217392, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.020023388788104057, |
| "learning_rate": 5e-05, |
| "loss": 0.010980712249875069, |
| "num_tokens": 5253571.0, |
| "reward": 0.42290037870407104, |
| "reward_std": 0.3821259140968323, |
| "rewards/correctness/mean": 0.7400000095367432, |
| "rewards/correctness/std": 0.44308748841285706, |
| "rewards/length_penalty/mean": -0.31709960103034973, |
| "rewards/length_penalty/std": 0.09378077834844589, |
| "sampling/importance_sampling_ratio/max": 2.707496404647827, |
| "sampling/importance_sampling_ratio/mean": 0.9943109154701233, |
| "sampling/importance_sampling_ratio/min": 0.11866816133260727, |
| "sampling/sampling_logp_difference/max": 2.1314241886138916, |
| "sampling/sampling_logp_difference/mean": 0.0196909848600626, |
| "step": 81, |
| "step_time": 13.374773803167045 |
| }, |
| { |
| "clip_ratio/high_max": 0.0003143954527331516, |
| "clip_ratio/high_mean": 0.0003143954527331516, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0003143954527331516, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 748.0, |
| "completions/max_terminated_length": 748.0, |
| "completions/mean_length": 444.5799865722656, |
| "completions/mean_terminated_length": 444.5799865722656, |
| "completions/min_length": 238.0, |
| "completions/min_terminated_length": 238.0, |
| "entropy": 0.1351422816514969, |
| "epoch": 0.22282608695652173, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.013173939660191536, |
| "learning_rate": 5e-05, |
| "loss": 0.020658979192376137, |
| "num_tokens": 5278650.0, |
| "reward": 0.5829198956489563, |
| "reward_std": 0.46361181139945984, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.21708007156848907, |
| "rewards/length_penalty/std": 0.06732401251792908, |
| "sampling/importance_sampling_ratio/max": 2.7757651805877686, |
| "sampling/importance_sampling_ratio/mean": 0.9944673776626587, |
| "sampling/importance_sampling_ratio/min": 0.31246867775917053, |
| "sampling/sampling_logp_difference/max": 1.1632510423660278, |
| "sampling/sampling_logp_difference/mean": 0.021512571722269058, |
| "step": 82, |
| "step_time": 9.464399790856987 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006479248695541173, |
| "clip_ratio/high_mean": 0.0006479248695541173, |
| "clip_ratio/low_mean": 0.00010048146359622478, |
| "clip_ratio/low_min": 0.00010048146359622478, |
| "clip_ratio/region_mean": 0.0007484063331503421, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 720.0, |
| "completions/max_terminated_length": 720.0, |
| "completions/mean_length": 408.0799865722656, |
| "completions/mean_terminated_length": 408.0799865722656, |
| "completions/min_length": 237.0, |
| "completions/min_terminated_length": 237.0, |
| "entropy": 0.1544148862361908, |
| "epoch": 0.22554347826086957, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.014162655919790268, |
| "learning_rate": 5e-05, |
| "loss": 0.009620768949389458, |
| "num_tokens": 5301484.0, |
| "reward": 0.3607421815395355, |
| "reward_std": 0.4975096583366394, |
| "rewards/correctness/mean": 0.5600000023841858, |
| "rewards/correctness/std": 0.5014265179634094, |
| "rewards/length_penalty/mean": -0.19925780594348907, |
| "rewards/length_penalty/std": 0.06029851362109184, |
| "sampling/importance_sampling_ratio/max": 2.475653648376465, |
| "sampling/importance_sampling_ratio/mean": 0.9939197301864624, |
| "sampling/importance_sampling_ratio/min": 0.22558780014514923, |
| "sampling/sampling_logp_difference/max": 1.4890458583831787, |
| "sampling/sampling_logp_difference/mean": 0.023020122200250626, |
| "step": 83, |
| "step_time": 8.50258247810416 |
| }, |
| { |
| "clip_ratio/high_max": 0.0009030202636495233, |
| "clip_ratio/high_mean": 0.0009030202636495233, |
| "clip_ratio/low_mean": 7.876795716583729e-05, |
| "clip_ratio/low_min": 7.876795716583729e-05, |
| "clip_ratio/region_mean": 0.0009817882324568926, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1560.0, |
| "completions/max_terminated_length": 1560.0, |
| "completions/mean_length": 530.8399658203125, |
| "completions/mean_terminated_length": 530.8399658203125, |
| "completions/min_length": 193.0, |
| "completions/min_terminated_length": 193.0, |
| "entropy": 0.14245042204856873, |
| "epoch": 0.22826086956521738, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.014098316431045532, |
| "learning_rate": 5e-05, |
| "loss": 0.028290294110774994, |
| "num_tokens": 5331006.0, |
| "reward": 0.5408007502555847, |
| "reward_std": 0.3885652720928192, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.25919923186302185, |
| "rewards/length_penalty/std": 0.1674625128507614, |
| "sampling/importance_sampling_ratio/max": 2.278921604156494, |
| "sampling/importance_sampling_ratio/mean": 0.9943795800209045, |
| "sampling/importance_sampling_ratio/min": 0.21966639161109924, |
| "sampling/sampling_logp_difference/max": 1.5156453847885132, |
| "sampling/sampling_logp_difference/mean": 0.019804934039711952, |
| "step": 84, |
| "step_time": 17.037076706066728 |
| }, |
| { |
| "clip_ratio/high_max": 0.0018277494236826897, |
| "clip_ratio/high_mean": 0.0018277494236826897, |
| "clip_ratio/low_mean": 0.00013214687642175703, |
| "clip_ratio/low_min": 0.00013214687642175703, |
| "clip_ratio/region_mean": 0.001959896294283681, |
| "completions/clipped_ratio": 0.03999999910593033, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 2027.0, |
| "completions/mean_length": 700.8399658203125, |
| "completions/mean_terminated_length": 644.7083740234375, |
| "completions/min_length": 203.0, |
| "completions/min_terminated_length": 203.0, |
| "entropy": 0.2856750160455704, |
| "epoch": 0.23097826086956522, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.026974737644195557, |
| "learning_rate": 5e-05, |
| "loss": 0.09304343163967133, |
| "num_tokens": 5368068.0, |
| "reward": 0.3377929627895355, |
| "reward_std": 0.5755784511566162, |
| "rewards/correctness/mean": 0.6800000071525574, |
| "rewards/correctness/std": 0.47121208906173706, |
| "rewards/length_penalty/mean": -0.34220704436302185, |
| "rewards/length_penalty/std": 0.2737504541873932, |
| "sampling/importance_sampling_ratio/max": 2.973176956176758, |
| "sampling/importance_sampling_ratio/mean": 0.988871157169342, |
| "sampling/importance_sampling_ratio/min": 0.13172246515750885, |
| "sampling/sampling_logp_difference/max": 2.0270581245422363, |
| "sampling/sampling_logp_difference/mean": 0.028557635843753815, |
| "step": 85, |
| "step_time": 22.43926484696567 |
| }, |
| { |
| "clip_ratio/high_max": 0.001044186291983351, |
| "clip_ratio/high_mean": 0.001044186291983351, |
| "clip_ratio/low_mean": 0.00011646859929896891, |
| "clip_ratio/low_min": 0.00011646859929896891, |
| "clip_ratio/region_mean": 0.0011606548796407878, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 527.0, |
| "completions/max_terminated_length": 527.0, |
| "completions/mean_length": 310.6600036621094, |
| "completions/mean_terminated_length": 310.6600036621094, |
| "completions/min_length": 151.0, |
| "completions/min_terminated_length": 151.0, |
| "entropy": 0.14149793088436127, |
| "epoch": 0.23369565217391305, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01278267614543438, |
| "learning_rate": 5e-05, |
| "loss": 0.01356387510895729, |
| "num_tokens": 5385701.0, |
| "reward": 0.8483105301856995, |
| "reward_std": 0.046941135078668594, |
| "rewards/correctness/mean": 1.0, |
| "rewards/correctness/std": 0.0, |
| "rewards/length_penalty/mean": -0.15168945491313934, |
| "rewards/length_penalty/std": 0.04694113880395889, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9945849776268005, |
| "sampling/importance_sampling_ratio/min": 0.2063879370689392, |
| "sampling/sampling_logp_difference/max": 1.5779976844787598, |
| "sampling/sampling_logp_difference/mean": 0.02506762556731701, |
| "step": 86, |
| "step_time": 6.345105629879981 |
| }, |
| { |
| "clip_ratio/high_max": 0.000682604851317592, |
| "clip_ratio/high_mean": 0.000682604851317592, |
| "clip_ratio/low_mean": 9.332875779364259e-05, |
| "clip_ratio/low_min": 9.332875779364259e-05, |
| "clip_ratio/region_mean": 0.0007759336091112346, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1127.0, |
| "completions/max_terminated_length": 1127.0, |
| "completions/mean_length": 498.41998291015625, |
| "completions/mean_terminated_length": 498.41998291015625, |
| "completions/min_length": 146.0, |
| "completions/min_terminated_length": 146.0, |
| "entropy": 0.14586662650108337, |
| "epoch": 0.23641304347826086, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0155003247782588, |
| "learning_rate": 5e-05, |
| "loss": 0.029746606945991516, |
| "num_tokens": 5412832.0, |
| "reward": 0.3566308617591858, |
| "reward_std": 0.560200035572052, |
| "rewards/correctness/mean": 0.6000000238418579, |
| "rewards/correctness/std": 0.4948716461658478, |
| "rewards/length_penalty/mean": -0.24336914718151093, |
| "rewards/length_penalty/std": 0.12490728497505188, |
| "sampling/importance_sampling_ratio/max": 2.270101308822632, |
| "sampling/importance_sampling_ratio/mean": 0.9942162036895752, |
| "sampling/importance_sampling_ratio/min": 0.2935054302215576, |
| "sampling/sampling_logp_difference/max": 1.2258591651916504, |
| "sampling/sampling_logp_difference/mean": 0.01999354176223278, |
| "step": 87, |
| "step_time": 13.222105751745403 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005343808559700846, |
| "clip_ratio/high_mean": 0.0005343808559700846, |
| "clip_ratio/low_mean": 0.00016299273702315986, |
| "clip_ratio/low_min": 0.00016299273702315986, |
| "clip_ratio/region_mean": 0.0006973735755309463, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 679.0, |
| "completions/max_terminated_length": 679.0, |
| "completions/mean_length": 382.0799865722656, |
| "completions/mean_terminated_length": 382.0799865722656, |
| "completions/min_length": 136.0, |
| "completions/min_terminated_length": 136.0, |
| "entropy": 0.13788711428642272, |
| "epoch": 0.2391304347826087, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.016201354563236237, |
| "learning_rate": 5e-05, |
| "loss": 0.008741471916437149, |
| "num_tokens": 5434826.0, |
| "reward": 0.7734375, |
| "reward_std": 0.2171802967786789, |
| "rewards/correctness/mean": 0.9599999785423279, |
| "rewards/correctness/std": 0.1979486644268036, |
| "rewards/length_penalty/mean": -0.18656249344348907, |
| "rewards/length_penalty/std": 0.06984082609415054, |
| "sampling/importance_sampling_ratio/max": 2.7883358001708984, |
| "sampling/importance_sampling_ratio/mean": 0.9944828152656555, |
| "sampling/importance_sampling_ratio/min": 0.1857496201992035, |
| "sampling/sampling_logp_difference/max": 1.6833555698394775, |
| "sampling/sampling_logp_difference/mean": 0.021422382444143295, |
| "step": 88, |
| "step_time": 8.28874902217649 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007955357184982858, |
| "clip_ratio/high_mean": 0.0007955357184982858, |
| "clip_ratio/low_mean": 2.621919265948236e-05, |
| "clip_ratio/low_min": 2.621919265948236e-05, |
| "clip_ratio/region_mean": 0.0008217549111577682, |
| "completions/clipped_ratio": 0.019999999552965164, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1909.0, |
| "completions/mean_length": 618.5, |
| "completions/mean_terminated_length": 589.3265380859375, |
| "completions/min_length": 197.0, |
| "completions/min_terminated_length": 197.0, |
| "entropy": 0.17175524830818176, |
| "epoch": 0.2418478260869565, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.013653785921633244, |
| "learning_rate": 5e-05, |
| "loss": 0.020935919135808945, |
| "num_tokens": 5469211.0, |
| "reward": 0.21799804270267487, |
| "reward_std": 0.6661584377288818, |
| "rewards/correctness/mean": 0.5199999809265137, |
| "rewards/correctness/std": 0.5046720504760742, |
| "rewards/length_penalty/mean": -0.302001953125, |
| "rewards/length_penalty/std": 0.2297671139240265, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9930328130722046, |
| "sampling/importance_sampling_ratio/min": 0.303615540266037, |
| "sampling/sampling_logp_difference/max": 1.191992998123169, |
| "sampling/sampling_logp_difference/mean": 0.020726434886455536, |
| "step": 89, |
| "step_time": 22.4951800100971 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007658500340767205, |
| "clip_ratio/high_mean": 0.0007658500340767205, |
| "clip_ratio/low_mean": 2.3438416246790438e-05, |
| "clip_ratio/low_min": 2.3438416246790438e-05, |
| "clip_ratio/region_mean": 0.0007892884546890855, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1998.0, |
| "completions/max_terminated_length": 1998.0, |
| "completions/mean_length": 659.739990234375, |
| "completions/mean_terminated_length": 659.739990234375, |
| "completions/min_length": 274.0, |
| "completions/min_terminated_length": 274.0, |
| "entropy": 0.14435225278139113, |
| "epoch": 0.24456521739130435, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.019286051392555237, |
| "learning_rate": 5e-05, |
| "loss": 0.03661634027957916, |
| "num_tokens": 5505228.0, |
| "reward": 0.6178613305091858, |
| "reward_std": 0.36470702290534973, |
| "rewards/correctness/mean": 0.9399999976158142, |
| "rewards/correctness/std": 0.2398979514837265, |
| "rewards/length_penalty/mean": -0.3221386671066284, |
| "rewards/length_penalty/std": 0.22656019032001495, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9943234920501709, |
| "sampling/importance_sampling_ratio/min": 0.3591086268424988, |
| "sampling/sampling_logp_difference/max": 1.1295256614685059, |
| "sampling/sampling_logp_difference/mean": 0.019089285284280777, |
| "step": 90, |
| "step_time": 21.950008247047663 |
| }, |
| { |
| "clip_ratio/high_max": 0.0009108542930334806, |
| "clip_ratio/high_mean": 0.0009108542930334806, |
| "clip_ratio/low_mean": 0.00010948879062198103, |
| "clip_ratio/low_min": 0.00010948879062198103, |
| "clip_ratio/region_mean": 0.0010203430894762278, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1604.0, |
| "completions/max_terminated_length": 1604.0, |
| "completions/mean_length": 517.0800170898438, |
| "completions/mean_terminated_length": 517.0800170898438, |
| "completions/min_length": 89.0, |
| "completions/min_terminated_length": 89.0, |
| "entropy": 0.17385907024145125, |
| "epoch": 0.24728260869565216, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017237825319170952, |
| "learning_rate": 5e-05, |
| "loss": 0.04393135383725166, |
| "num_tokens": 5536252.0, |
| "reward": 0.6875194907188416, |
| "reward_std": 0.365161269903183, |
| "rewards/correctness/mean": 0.9399999976158142, |
| "rewards/correctness/std": 0.2398979365825653, |
| "rewards/length_penalty/mean": -0.25248047709465027, |
| "rewards/length_penalty/std": 0.1945510357618332, |
| "sampling/importance_sampling_ratio/max": 2.810533046722412, |
| "sampling/importance_sampling_ratio/mean": 0.9928355813026428, |
| "sampling/importance_sampling_ratio/min": 0.2914602756500244, |
| "sampling/sampling_logp_difference/max": 1.232851505279541, |
| "sampling/sampling_logp_difference/mean": 0.022177454084157944, |
| "step": 91, |
| "step_time": 18.435933465138078 |
| }, |
| { |
| "clip_ratio/high_max": 0.00046310991747304796, |
| "clip_ratio/high_mean": 0.00046310991747304796, |
| "clip_ratio/low_mean": 0.00012931099627166986, |
| "clip_ratio/low_min": 0.00012931099627166986, |
| "clip_ratio/region_mean": 0.000592420919565484, |
| "completions/clipped_ratio": 0.019999999552965164, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1596.0, |
| "completions/mean_length": 511.2200012207031, |
| "completions/mean_terminated_length": 479.8571472167969, |
| "completions/min_length": 156.0, |
| "completions/min_terminated_length": 156.0, |
| "entropy": 0.14240270107984543, |
| "epoch": 0.25, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.014772976748645306, |
| "learning_rate": 5e-05, |
| "loss": 0.042228199541568756, |
| "num_tokens": 5564203.0, |
| "reward": 0.5503808259963989, |
| "reward_std": 0.42025887966156006, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.40406104922294617, |
| "rewards/length_penalty/mean": -0.24961914122104645, |
| "rewards/length_penalty/std": 0.2091631144285202, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9942843317985535, |
| "sampling/importance_sampling_ratio/min": 0.18323083221912384, |
| "sampling/sampling_logp_difference/max": 1.6970086097717285, |
| "sampling/sampling_logp_difference/mean": 0.021515555679798126, |
| "step": 92, |
| "step_time": 22.50430703512393 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007045186866889708, |
| "clip_ratio/high_mean": 0.0007045186866889708, |
| "clip_ratio/low_mean": 0.0002806035408866592, |
| "clip_ratio/low_min": 0.0002806035408866592, |
| "clip_ratio/region_mean": 0.0009851222421275452, |
| "completions/clipped_ratio": 0.05999999865889549, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1638.0, |
| "completions/mean_length": 669.0999755859375, |
| "completions/mean_terminated_length": 581.0850830078125, |
| "completions/min_length": 187.0, |
| "completions/min_terminated_length": 187.0, |
| "entropy": 0.20341256856918336, |
| "epoch": 0.25271739130434784, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.020352164283394814, |
| "learning_rate": 5e-05, |
| "loss": 0.10169492661952972, |
| "num_tokens": 5603278.0, |
| "reward": 0.4732910096645355, |
| "reward_std": 0.5773070454597473, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.32670897245407104, |
| "rewards/length_penalty/std": 0.2420545071363449, |
| "sampling/importance_sampling_ratio/max": 2.4024431705474854, |
| "sampling/importance_sampling_ratio/mean": 0.9923534393310547, |
| "sampling/importance_sampling_ratio/min": 0.2638809382915497, |
| "sampling/sampling_logp_difference/max": 1.3322572708129883, |
| "sampling/sampling_logp_difference/mean": 0.021952515468001366, |
| "step": 93, |
| "step_time": 23.539885837351903 |
| }, |
| { |
| "clip_ratio/high_max": 0.0011616522213444113, |
| "clip_ratio/high_mean": 0.0011616522213444113, |
| "clip_ratio/low_mean": 0.0002181586460210383, |
| "clip_ratio/low_min": 0.0002181586460210383, |
| "clip_ratio/region_mean": 0.0013798108557239175, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1176.0, |
| "completions/max_terminated_length": 1176.0, |
| "completions/mean_length": 450.239990234375, |
| "completions/mean_terminated_length": 450.239990234375, |
| "completions/min_length": 215.0, |
| "completions/min_terminated_length": 215.0, |
| "entropy": 0.19916799068450927, |
| "epoch": 0.2554347826086957, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01750258356332779, |
| "learning_rate": 5e-05, |
| "loss": 0.0182659812271595, |
| "num_tokens": 5631820.0, |
| "reward": 0.30015623569488525, |
| "reward_std": 0.461910218000412, |
| "rewards/correctness/mean": 0.5199999809265137, |
| "rewards/correctness/std": 0.5046720504760742, |
| "rewards/length_penalty/mean": -0.21984374523162842, |
| "rewards/length_penalty/std": 0.10487209260463715, |
| "sampling/importance_sampling_ratio/max": 2.1481571197509766, |
| "sampling/importance_sampling_ratio/mean": 0.991895854473114, |
| "sampling/importance_sampling_ratio/min": 0.24272750318050385, |
| "sampling/sampling_logp_difference/max": 1.415815830230713, |
| "sampling/sampling_logp_difference/mean": 0.025588279590010643, |
| "step": 94, |
| "step_time": 13.953644986730069 |
| }, |
| { |
| "clip_ratio/high_max": 0.0011509397649206222, |
| "clip_ratio/high_mean": 0.0011509397649206222, |
| "clip_ratio/low_mean": 0.00019863411434926093, |
| "clip_ratio/low_min": 0.00019863411434926093, |
| "clip_ratio/region_mean": 0.0013495738967321813, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 459.0, |
| "completions/max_terminated_length": 459.0, |
| "completions/mean_length": 290.0799865722656, |
| "completions/mean_terminated_length": 290.0799865722656, |
| "completions/min_length": 155.0, |
| "completions/min_terminated_length": 155.0, |
| "entropy": 0.1961733102798462, |
| "epoch": 0.25815217391304346, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01650562509894371, |
| "learning_rate": 5e-05, |
| "loss": 0.009883588179945946, |
| "num_tokens": 5648694.0, |
| "reward": 0.8383593559265137, |
| "reward_std": 0.14430737495422363, |
| "rewards/correctness/mean": 0.9800000190734863, |
| "rewards/correctness/std": 0.1414213478565216, |
| "rewards/length_penalty/mean": -0.14164061844348907, |
| "rewards/length_penalty/std": 0.03877919539809227, |
| "sampling/importance_sampling_ratio/max": 2.824411630630493, |
| "sampling/importance_sampling_ratio/mean": 0.9931517839431763, |
| "sampling/importance_sampling_ratio/min": 0.35756808519363403, |
| "sampling/sampling_logp_difference/max": 1.0383000373840332, |
| "sampling/sampling_logp_difference/mean": 0.030635839328169823, |
| "step": 95, |
| "step_time": 5.755662291543558 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006952405674383044, |
| "clip_ratio/high_mean": 0.0006952405674383044, |
| "clip_ratio/low_mean": 0.0003287168277893215, |
| "clip_ratio/low_min": 0.0003287168277893215, |
| "clip_ratio/region_mean": 0.0010239573777653277, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 684.0, |
| "completions/max_terminated_length": 684.0, |
| "completions/mean_length": 312.79998779296875, |
| "completions/mean_terminated_length": 312.79998779296875, |
| "completions/min_length": 101.0, |
| "completions/min_terminated_length": 101.0, |
| "entropy": 0.18830790221691132, |
| "epoch": 0.2608695652173913, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0161149799823761, |
| "learning_rate": 5e-05, |
| "loss": 0.014134292490780354, |
| "num_tokens": 5667864.0, |
| "reward": 0.5272656083106995, |
| "reward_std": 0.5117128491401672, |
| "rewards/correctness/mean": 0.6800000071525574, |
| "rewards/correctness/std": 0.47121208906173706, |
| "rewards/length_penalty/mean": -0.15273436903953552, |
| "rewards/length_penalty/std": 0.0754011869430542, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9926605820655823, |
| "sampling/importance_sampling_ratio/min": 0.2316138744354248, |
| "sampling/sampling_logp_difference/max": 1.4626836776733398, |
| "sampling/sampling_logp_difference/mean": 0.0293266698718071, |
| "step": 96, |
| "step_time": 8.08534470712766 |
| }, |
| { |
| "clip_ratio/high_max": 0.0009365640406031161, |
| "clip_ratio/high_mean": 0.0009365640406031161, |
| "clip_ratio/low_mean": 7.815553108230233e-05, |
| "clip_ratio/low_min": 7.815553108230233e-05, |
| "clip_ratio/region_mean": 0.0010147195833269506, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 546.0, |
| "completions/max_terminated_length": 546.0, |
| "completions/mean_length": 302.9599914550781, |
| "completions/mean_terminated_length": 302.9599914550781, |
| "completions/min_length": 133.0, |
| "completions/min_terminated_length": 133.0, |
| "entropy": 0.19039133489131926, |
| "epoch": 0.26358695652173914, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01323253195732832, |
| "learning_rate": 5e-05, |
| "loss": 0.007963886484503746, |
| "num_tokens": 5685942.0, |
| "reward": 0.8320702910423279, |
| "reward_std": 0.14255400002002716, |
| "rewards/correctness/mean": 0.9800000190734863, |
| "rewards/correctness/std": 0.1414213478565216, |
| "rewards/length_penalty/mean": -0.14792968332767487, |
| "rewards/length_penalty/std": 0.053403496742248535, |
| "sampling/importance_sampling_ratio/max": 2.9804489612579346, |
| "sampling/importance_sampling_ratio/mean": 0.9915622472763062, |
| "sampling/importance_sampling_ratio/min": 0.23956701159477234, |
| "sampling/sampling_logp_difference/max": 1.428922176361084, |
| "sampling/sampling_logp_difference/mean": 0.029504768550395966, |
| "step": 97, |
| "step_time": 7.233770550927147 |
| }, |
| { |
| "clip_ratio/high_max": 0.0015867566457018257, |
| "clip_ratio/high_mean": 0.0015867566457018257, |
| "clip_ratio/low_mean": 0.00011946987069677562, |
| "clip_ratio/low_min": 0.00011946987069677562, |
| "clip_ratio/region_mean": 0.0017062265193089842, |
| "completions/clipped_ratio": 0.05999999865889549, |
| "completions/max_length": 2048.0, |
| "completions/max_terminated_length": 1352.0, |
| "completions/mean_length": 349.7599792480469, |
| "completions/mean_terminated_length": 241.3616943359375, |
| "completions/min_length": 74.0, |
| "completions/min_terminated_length": 74.0, |
| "entropy": 0.22566365003585814, |
| "epoch": 0.266304347826087, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.015148256905376911, |
| "learning_rate": 5e-05, |
| "loss": 0.0725148394703865, |
| "num_tokens": 5705570.0, |
| "reward": 0.2292187511920929, |
| "reward_std": 0.6269381046295166, |
| "rewards/correctness/mean": 0.4000000059604645, |
| "rewards/correctness/std": 0.4948716461658478, |
| "rewards/length_penalty/mean": -0.17078125476837158, |
| "rewards/length_penalty/std": 0.24175436794757843, |
| "sampling/importance_sampling_ratio/max": 2.5255091190338135, |
| "sampling/importance_sampling_ratio/mean": 0.991366982460022, |
| "sampling/importance_sampling_ratio/min": 0.09896191209554672, |
| "sampling/sampling_logp_difference/max": 2.3130202293395996, |
| "sampling/sampling_logp_difference/mean": 0.026597710326313972, |
| "step": 98, |
| "step_time": 21.33166946657002 |
| }, |
| { |
| "clip_ratio/high_max": 0.0010735183954238892, |
| "clip_ratio/high_mean": 0.0010735183954238892, |
| "clip_ratio/low_mean": 0.00030840777326375247, |
| "clip_ratio/low_min": 0.00030840777326375247, |
| "clip_ratio/region_mean": 0.0013819261686876417, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 310.0, |
| "completions/max_terminated_length": 310.0, |
| "completions/mean_length": 185.86000061035156, |
| "completions/mean_terminated_length": 185.86000061035156, |
| "completions/min_length": 92.0, |
| "completions/min_terminated_length": 92.0, |
| "entropy": 0.20395126342773437, |
| "epoch": 0.26902173913043476, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012664223089814186, |
| "learning_rate": 5e-05, |
| "loss": 0.007070655468851328, |
| "num_tokens": 5717773.0, |
| "reward": 0.6292480230331421, |
| "reward_std": 0.46103474497795105, |
| "rewards/correctness/mean": 0.7200000286102295, |
| "rewards/correctness/std": 0.4535573720932007, |
| "rewards/length_penalty/mean": -0.09075195342302322, |
| "rewards/length_penalty/std": 0.024525616317987442, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9933871626853943, |
| "sampling/importance_sampling_ratio/min": 0.20495867729187012, |
| "sampling/sampling_logp_difference/max": 1.584946870803833, |
| "sampling/sampling_logp_difference/mean": 0.033781420439481735, |
| "step": 99, |
| "step_time": 4.117180567001924 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005707273143343627, |
| "clip_ratio/high_mean": 0.0005707273143343627, |
| "clip_ratio/low_mean": 9.760858956724406e-05, |
| "clip_ratio/low_min": 9.760858956724406e-05, |
| "clip_ratio/region_mean": 0.0006683359039016068, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 346.0, |
| "completions/max_terminated_length": 346.0, |
| "completions/mean_length": 187.25999450683594, |
| "completions/mean_terminated_length": 187.25999450683594, |
| "completions/min_length": 77.0, |
| "completions/min_terminated_length": 77.0, |
| "entropy": 0.17016780972480774, |
| "epoch": 0.2717391304347826, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012072809040546417, |
| "learning_rate": 5e-05, |
| "loss": 0.008675863966345787, |
| "num_tokens": 5729366.0, |
| "reward": 0.88856440782547, |
| "reward_std": 0.14345325529575348, |
| "rewards/correctness/mean": 0.9800000190734863, |
| "rewards/correctness/std": 0.1414213478565216, |
| "rewards/length_penalty/mean": -0.09143554419279099, |
| "rewards/length_penalty/std": 0.03134666755795479, |
| "sampling/importance_sampling_ratio/max": 2.592597246170044, |
| "sampling/importance_sampling_ratio/mean": 0.991979718208313, |
| "sampling/importance_sampling_ratio/min": 0.28697964549064636, |
| "sampling/sampling_logp_difference/max": 1.2483439445495605, |
| "sampling/sampling_logp_difference/mean": 0.03182890638709068, |
| "step": 100, |
| "step_time": 4.314533639000729 |
| }, |
| { |
| "clip_ratio/high_max": 0.0012083934037946165, |
| "clip_ratio/high_mean": 0.0012083934037946165, |
| "clip_ratio/low_mean": 0.00026164305745624006, |
| "clip_ratio/low_min": 0.00026164305745624006, |
| "clip_ratio/region_mean": 0.0014700364205054938, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 452.0, |
| "completions/max_terminated_length": 452.0, |
| "completions/mean_length": 204.6599884033203, |
| "completions/mean_terminated_length": 204.6599884033203, |
| "completions/min_length": 96.0, |
| "completions/min_terminated_length": 96.0, |
| "entropy": 0.19577673971652984, |
| "epoch": 0.27445652173913043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.013354828581213951, |
| "learning_rate": 5e-05, |
| "loss": 0.012918182648718357, |
| "num_tokens": 5742489.0, |
| "reward": 0.8400683403015137, |
| "reward_std": 0.24915407598018646, |
| "rewards/correctness/mean": 0.9399999976158142, |
| "rewards/correctness/std": 0.2398979365825653, |
| "rewards/length_penalty/mean": -0.09993164241313934, |
| "rewards/length_penalty/std": 0.04284634813666344, |
| "sampling/importance_sampling_ratio/max": 2.8144924640655518, |
| "sampling/importance_sampling_ratio/mean": 0.9925035834312439, |
| "sampling/importance_sampling_ratio/min": 0.19658847153186798, |
| "sampling/sampling_logp_difference/max": 1.6266427040100098, |
| "sampling/sampling_logp_difference/mean": 0.03311571851372719, |
| "step": 101, |
| "step_time": 5.483416471164674 |
| }, |
| { |
| "clip_ratio/high_max": 0.0009411082835868001, |
| "clip_ratio/high_mean": 0.0009411082835868001, |
| "clip_ratio/low_mean": 0.00022402363829314708, |
| "clip_ratio/low_min": 0.00022402363829314708, |
| "clip_ratio/region_mean": 0.0011651319218799472, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 796.0, |
| "completions/max_terminated_length": 796.0, |
| "completions/mean_length": 234.13999938964844, |
| "completions/mean_terminated_length": 234.13999938964844, |
| "completions/min_length": 50.0, |
| "completions/min_terminated_length": 50.0, |
| "entropy": 0.2318212479352951, |
| "epoch": 0.27717391304347827, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.014598333276808262, |
| "learning_rate": 5e-05, |
| "loss": -0.0033794669434428215, |
| "num_tokens": 5757126.0, |
| "reward": 0.6456738114356995, |
| "reward_std": 0.48349666595458984, |
| "rewards/correctness/mean": 0.7599999904632568, |
| "rewards/correctness/std": 0.43141913414001465, |
| "rewards/length_penalty/mean": -0.11432617157697678, |
| "rewards/length_penalty/std": 0.08932963013648987, |
| "sampling/importance_sampling_ratio/max": 2.6214189529418945, |
| "sampling/importance_sampling_ratio/mean": 0.989772379398346, |
| "sampling/importance_sampling_ratio/min": 0.1754714399576187, |
| "sampling/sampling_logp_difference/max": 1.740278959274292, |
| "sampling/sampling_logp_difference/mean": 0.034885525703430176, |
| "step": 102, |
| "step_time": 8.834860250819474 |
| }, |
| { |
| "clip_ratio/high_max": 0.0014411504263989628, |
| "clip_ratio/high_mean": 0.0014411504263989628, |
| "clip_ratio/low_mean": 9.528346708975732e-05, |
| "clip_ratio/low_min": 9.528346708975732e-05, |
| "clip_ratio/region_mean": 0.0015364338527433575, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 348.0, |
| "completions/max_terminated_length": 348.0, |
| "completions/mean_length": 174.37998962402344, |
| "completions/mean_terminated_length": 174.37998962402344, |
| "completions/min_length": 71.0, |
| "completions/min_terminated_length": 71.0, |
| "entropy": 0.2715637177228928, |
| "epoch": 0.2798913043478261, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017685387283563614, |
| "learning_rate": 5e-05, |
| "loss": 0.01238732598721981, |
| "num_tokens": 5770085.0, |
| "reward": 0.7148535251617432, |
| "reward_std": 0.427943617105484, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.08514648675918579, |
| "rewards/length_penalty/std": 0.03423784300684929, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9892432689666748, |
| "sampling/importance_sampling_ratio/min": 0.2573729157447815, |
| "sampling/sampling_logp_difference/max": 1.357229232788086, |
| "sampling/sampling_logp_difference/mean": 0.038839783519506454, |
| "step": 103, |
| "step_time": 4.841475100722164 |
| }, |
| { |
| "clip_ratio/high_max": 0.0018297017086297274, |
| "clip_ratio/high_mean": 0.0018297017086297274, |
| "clip_ratio/low_mean": 0.00037032614927738904, |
| "clip_ratio/low_min": 0.00037032614927738904, |
| "clip_ratio/region_mean": 0.002200027834624052, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 930.0, |
| "completions/max_terminated_length": 930.0, |
| "completions/mean_length": 327.6000061035156, |
| "completions/mean_terminated_length": 327.6000061035156, |
| "completions/min_length": 79.0, |
| "completions/min_terminated_length": 79.0, |
| "entropy": 0.29253464937210083, |
| "epoch": 0.2826086956521739, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.017163075506687164, |
| "learning_rate": 5e-05, |
| "loss": 0.02614499069750309, |
| "num_tokens": 5789975.0, |
| "reward": 0.2000390589237213, |
| "reward_std": 0.5057170987129211, |
| "rewards/correctness/mean": 0.36000001430511475, |
| "rewards/correctness/std": 0.4848732352256775, |
| "rewards/length_penalty/mean": -0.15996094048023224, |
| "rewards/length_penalty/std": 0.09843949228525162, |
| "sampling/importance_sampling_ratio/max": 2.5560290813446045, |
| "sampling/importance_sampling_ratio/mean": 0.9881027936935425, |
| "sampling/importance_sampling_ratio/min": 0.21785800158977509, |
| "sampling/sampling_logp_difference/max": 1.5239118337631226, |
| "sampling/sampling_logp_difference/mean": 0.03395746275782585, |
| "step": 104, |
| "step_time": 10.365797318983823 |
| }, |
| { |
| "clip_ratio/high_max": 0.0016505021136254071, |
| "clip_ratio/high_mean": 0.0016505021136254071, |
| "clip_ratio/low_mean": 0.0004149336367845535, |
| "clip_ratio/low_min": 0.0004149336367845535, |
| "clip_ratio/region_mean": 0.0020654357504099607, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 525.0, |
| "completions/max_terminated_length": 525.0, |
| "completions/mean_length": 185.8199920654297, |
| "completions/mean_terminated_length": 185.8199920654297, |
| "completions/min_length": 76.0, |
| "completions/min_terminated_length": 76.0, |
| "entropy": 0.23983452320098878, |
| "epoch": 0.28532608695652173, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012423444539308548, |
| "learning_rate": 5e-05, |
| "loss": 0.002859638072550297, |
| "num_tokens": 5801626.0, |
| "reward": 0.38926756381988525, |
| "reward_std": 0.5165557265281677, |
| "rewards/correctness/mean": 0.47999998927116394, |
| "rewards/correctness/std": 0.5046720504760742, |
| "rewards/length_penalty/mean": -0.09073242545127869, |
| "rewards/length_penalty/std": 0.04708758741617203, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9901142716407776, |
| "sampling/importance_sampling_ratio/min": 0.17719721794128418, |
| "sampling/sampling_logp_difference/max": 1.7304918766021729, |
| "sampling/sampling_logp_difference/mean": 0.03442460298538208, |
| "step": 105, |
| "step_time": 6.492382062831894 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005780933774076402, |
| "clip_ratio/high_mean": 0.0005780933774076402, |
| "clip_ratio/low_mean": 0.0005185882211662829, |
| "clip_ratio/low_min": 0.0005185882211662829, |
| "clip_ratio/region_mean": 0.001096681598573923, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 565.0, |
| "completions/max_terminated_length": 565.0, |
| "completions/mean_length": 158.51998901367188, |
| "completions/mean_terminated_length": 158.51998901367188, |
| "completions/min_length": 64.0, |
| "completions/min_terminated_length": 64.0, |
| "entropy": 0.22457527220249177, |
| "epoch": 0.28804347826086957, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.014758552424609661, |
| "learning_rate": 5e-05, |
| "loss": -0.0042253821156919, |
| "num_tokens": 5812032.0, |
| "reward": 0.7225976586341858, |
| "reward_std": 0.43400877714157104, |
| "rewards/correctness/mean": 0.800000011920929, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.07740234583616257, |
| "rewards/length_penalty/std": 0.054773226380348206, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9911705255508423, |
| "sampling/importance_sampling_ratio/min": 0.29393672943115234, |
| "sampling/sampling_logp_difference/max": 1.2798686027526855, |
| "sampling/sampling_logp_difference/mean": 0.03353006765246391, |
| "step": 106, |
| "step_time": 6.204408699879423 |
| }, |
| { |
| "clip_ratio/high_max": 0.0013835940160788595, |
| "clip_ratio/high_mean": 0.0013835940160788595, |
| "clip_ratio/low_mean": 0.0005407711723819375, |
| "clip_ratio/low_min": 0.0005407711723819375, |
| "clip_ratio/region_mean": 0.0019243651768192649, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1896.0, |
| "completions/max_terminated_length": 1896.0, |
| "completions/mean_length": 377.41998291015625, |
| "completions/mean_terminated_length": 377.41998291015625, |
| "completions/min_length": 77.0, |
| "completions/min_terminated_length": 77.0, |
| "entropy": 0.3654423654079437, |
| "epoch": 0.2907608695652174, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.016043387353420258, |
| "learning_rate": 5e-05, |
| "loss": 0.030646812170743942, |
| "num_tokens": 5834943.0, |
| "reward": 0.33571287989616394, |
| "reward_std": 0.6409838795661926, |
| "rewards/correctness/mean": 0.5199999809265137, |
| "rewards/correctness/std": 0.5046720504760742, |
| "rewards/length_penalty/mean": -0.18428711593151093, |
| "rewards/length_penalty/std": 0.21686908602714539, |
| "sampling/importance_sampling_ratio/max": 2.1414754390716553, |
| "sampling/importance_sampling_ratio/mean": 0.9864636063575745, |
| "sampling/importance_sampling_ratio/min": 0.16718970239162445, |
| "sampling/sampling_logp_difference/max": 1.7886261940002441, |
| "sampling/sampling_logp_difference/mean": 0.03430148586630821, |
| "step": 107, |
| "step_time": 20.311360628111288 |
| }, |
| { |
| "clip_ratio/high_max": 0.0022310700500383974, |
| "clip_ratio/high_mean": 0.0022310700500383974, |
| "clip_ratio/low_mean": 0.0008199853007681667, |
| "clip_ratio/low_min": 0.0008199853007681667, |
| "clip_ratio/region_mean": 0.003051055292598903, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 315.0, |
| "completions/max_terminated_length": 315.0, |
| "completions/mean_length": 151.67999267578125, |
| "completions/mean_terminated_length": 151.67999267578125, |
| "completions/min_length": 60.0, |
| "completions/min_terminated_length": 60.0, |
| "entropy": 0.2812129735946655, |
| "epoch": 0.29347826086956524, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01154936384409666, |
| "learning_rate": 5e-05, |
| "loss": 0.00485996576026082, |
| "num_tokens": 5845577.0, |
| "reward": 0.4059374928474426, |
| "reward_std": 0.5127573013305664, |
| "rewards/correctness/mean": 0.47999998927116394, |
| "rewards/correctness/std": 0.5046720504760742, |
| "rewards/length_penalty/mean": -0.07406249642372131, |
| "rewards/length_penalty/std": 0.028034603223204613, |
| "sampling/importance_sampling_ratio/max": 2.603525161743164, |
| "sampling/importance_sampling_ratio/mean": 0.9909453988075256, |
| "sampling/importance_sampling_ratio/min": 0.1587897688150406, |
| "sampling/sampling_logp_difference/max": 1.8401741981506348, |
| "sampling/sampling_logp_difference/mean": 0.04085557535290718, |
| "step": 108, |
| "step_time": 4.2213947479613125 |
| }, |
| { |
| "clip_ratio/high_max": 0.001480993302538991, |
| "clip_ratio/high_mean": 0.001480993302538991, |
| "clip_ratio/low_mean": 0.00021252454607747495, |
| "clip_ratio/low_min": 0.00021252454607747495, |
| "clip_ratio/region_mean": 0.0016935178078711032, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1066.0, |
| "completions/max_terminated_length": 1066.0, |
| "completions/mean_length": 164.4199981689453, |
| "completions/mean_terminated_length": 164.4199981689453, |
| "completions/min_length": 67.0, |
| "completions/min_terminated_length": 67.0, |
| "entropy": 0.31263987720012665, |
| "epoch": 0.296195652173913, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012847491540014744, |
| "learning_rate": 5e-05, |
| "loss": 0.010743032209575176, |
| "num_tokens": 5856138.0, |
| "reward": 0.5997167825698853, |
| "reward_std": 0.520103394985199, |
| "rewards/correctness/mean": 0.6800000071525574, |
| "rewards/correctness/std": 0.47121208906173706, |
| "rewards/length_penalty/mean": -0.08028320223093033, |
| "rewards/length_penalty/std": 0.0816592127084732, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9857426881790161, |
| "sampling/importance_sampling_ratio/min": 0.2710735499858856, |
| "sampling/sampling_logp_difference/max": 1.3053652048110962, |
| "sampling/sampling_logp_difference/mean": 0.04090895876288414, |
| "step": 109, |
| "step_time": 10.641943448223174 |
| }, |
| { |
| "clip_ratio/high_max": 0.0023372828494757415, |
| "clip_ratio/high_mean": 0.0023372828494757415, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0023372828494757415, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 224.0, |
| "completions/max_terminated_length": 224.0, |
| "completions/mean_length": 121.57999420166016, |
| "completions/mean_terminated_length": 121.57999420166016, |
| "completions/min_length": 42.0, |
| "completions/min_terminated_length": 42.0, |
| "entropy": 0.25978425741195676, |
| "epoch": 0.29891304347826086, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012036431580781937, |
| "learning_rate": 5e-05, |
| "loss": 0.0021082288585603237, |
| "num_tokens": 5864797.0, |
| "reward": 0.8406347632408142, |
| "reward_std": 0.3036022186279297, |
| "rewards/correctness/mean": 0.8999999761581421, |
| "rewards/correctness/std": 0.30304577946662903, |
| "rewards/length_penalty/mean": -0.05936523526906967, |
| "rewards/length_penalty/std": 0.025582697242498398, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9901321530342102, |
| "sampling/importance_sampling_ratio/min": 0.21720051765441895, |
| "sampling/sampling_logp_difference/max": 1.5269343852996826, |
| "sampling/sampling_logp_difference/mean": 0.04141371324658394, |
| "step": 110, |
| "step_time": 3.1361711190547794 |
| }, |
| { |
| "clip_ratio/high_max": 0.0003656261716969311, |
| "clip_ratio/high_mean": 0.0003656261716969311, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0003656261716969311, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 285.0, |
| "completions/max_terminated_length": 285.0, |
| "completions/mean_length": 98.81999969482422, |
| "completions/mean_terminated_length": 98.81999969482422, |
| "completions/min_length": 33.0, |
| "completions/min_terminated_length": 33.0, |
| "entropy": 0.19361501634120942, |
| "epoch": 0.3016304347826087, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012895033694803715, |
| "learning_rate": 5e-05, |
| "loss": 0.00398594792932272, |
| "num_tokens": 5872398.0, |
| "reward": 0.6717480421066284, |
| "reward_std": 0.45583122968673706, |
| "rewards/correctness/mean": 0.7200000286102295, |
| "rewards/correctness/std": 0.4535573720932007, |
| "rewards/length_penalty/mean": -0.04825195297598839, |
| "rewards/length_penalty/std": 0.02495020627975464, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.992006242275238, |
| "sampling/importance_sampling_ratio/min": 0.15148277580738068, |
| "sampling/sampling_logp_difference/max": 1.8872833251953125, |
| "sampling/sampling_logp_difference/mean": 0.03226413577795029, |
| "step": 111, |
| "step_time": 3.619885540334508 |
| }, |
| { |
| "clip_ratio/high_max": 0.0004390946705825627, |
| "clip_ratio/high_mean": 0.0004390946705825627, |
| "clip_ratio/low_mean": 0.00033824756974354386, |
| "clip_ratio/low_min": 0.00033824756974354386, |
| "clip_ratio/region_mean": 0.0007773422403261065, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 578.0, |
| "completions/max_terminated_length": 578.0, |
| "completions/mean_length": 136.27999877929688, |
| "completions/mean_terminated_length": 136.27999877929688, |
| "completions/min_length": 50.0, |
| "completions/min_terminated_length": 50.0, |
| "entropy": 0.32020220458507537, |
| "epoch": 0.30434782608695654, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012484963983297348, |
| "learning_rate": 5e-05, |
| "loss": 0.009005909785628319, |
| "num_tokens": 5881512.0, |
| "reward": 0.31345701217651367, |
| "reward_std": 0.5048692226409912, |
| "rewards/correctness/mean": 0.3799999952316284, |
| "rewards/correctness/std": 0.4903143346309662, |
| "rewards/length_penalty/mean": -0.06654296815395355, |
| "rewards/length_penalty/std": 0.05672767013311386, |
| "sampling/importance_sampling_ratio/max": 2.9808573722839355, |
| "sampling/importance_sampling_ratio/mean": 0.9861035346984863, |
| "sampling/importance_sampling_ratio/min": 0.22736811637878418, |
| "sampling/sampling_logp_difference/max": 1.481184959411621, |
| "sampling/sampling_logp_difference/mean": 0.037614136934280396, |
| "step": 112, |
| "step_time": 6.328286922769621 |
| }, |
| { |
| "clip_ratio/high_max": 0.0008063952205702663, |
| "clip_ratio/high_mean": 0.0008063952205702663, |
| "clip_ratio/low_mean": 0.0002684563864022493, |
| "clip_ratio/low_min": 0.0002684563864022493, |
| "clip_ratio/region_mean": 0.0010748516069725156, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 159.0, |
| "completions/max_terminated_length": 159.0, |
| "completions/mean_length": 76.43999481201172, |
| "completions/mean_terminated_length": 76.43999481201172, |
| "completions/min_length": 30.0, |
| "completions/min_terminated_length": 30.0, |
| "entropy": 0.2393881142139435, |
| "epoch": 0.3070652173913043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.009909100830554962, |
| "learning_rate": 5e-05, |
| "loss": 0.0011619285214692354, |
| "num_tokens": 5888644.0, |
| "reward": 0.6426757574081421, |
| "reward_std": 0.47860875725746155, |
| "rewards/correctness/mean": 0.6800000071525574, |
| "rewards/correctness/std": 0.47121208906173706, |
| "rewards/length_penalty/mean": -0.037324219942092896, |
| "rewards/length_penalty/std": 0.013797693885862827, |
| "sampling/importance_sampling_ratio/max": 2.7209415435791016, |
| "sampling/importance_sampling_ratio/mean": 0.9932952523231506, |
| "sampling/importance_sampling_ratio/min": 0.3156265318393707, |
| "sampling/sampling_logp_difference/max": 1.1531956195831299, |
| "sampling/sampling_logp_difference/mean": 0.03954983130097389, |
| "step": 113, |
| "step_time": 2.577970744809136 |
| }, |
| { |
| "clip_ratio/high_max": 0.0017212016391567886, |
| "clip_ratio/high_mean": 0.0017212016391567886, |
| "clip_ratio/low_mean": 0.00017621145816519855, |
| "clip_ratio/low_min": 0.00017621145816519855, |
| "clip_ratio/region_mean": 0.0018974130623973906, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 359.0, |
| "completions/max_terminated_length": 359.0, |
| "completions/mean_length": 99.95999908447266, |
| "completions/mean_terminated_length": 99.95999908447266, |
| "completions/min_length": 46.0, |
| "completions/min_terminated_length": 46.0, |
| "entropy": 0.27610541582107545, |
| "epoch": 0.30978260869565216, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.014195474795997143, |
| "learning_rate": 5e-05, |
| "loss": 0.0016526561230421066, |
| "num_tokens": 5896182.0, |
| "reward": 0.6311913728713989, |
| "reward_std": 0.48117876052856445, |
| "rewards/correctness/mean": 0.6800000071525574, |
| "rewards/correctness/std": 0.47121208906173706, |
| "rewards/length_penalty/mean": -0.048808593302965164, |
| "rewards/length_penalty/std": 0.027750348672270775, |
| "sampling/importance_sampling_ratio/max": 1.9509680271148682, |
| "sampling/importance_sampling_ratio/mean": 0.9879252314567566, |
| "sampling/importance_sampling_ratio/min": 0.34516289830207825, |
| "sampling/sampling_logp_difference/max": 1.0637388229370117, |
| "sampling/sampling_logp_difference/mean": 0.036363665014505386, |
| "step": 114, |
| "step_time": 4.703229781938717 |
| }, |
| { |
| "clip_ratio/high_max": 0.002066467539407313, |
| "clip_ratio/high_mean": 0.002066467539407313, |
| "clip_ratio/low_mean": 0.00025575447361916306, |
| "clip_ratio/low_min": 0.00025575447361916306, |
| "clip_ratio/region_mean": 0.002322222013026476, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 176.0, |
| "completions/max_terminated_length": 176.0, |
| "completions/mean_length": 80.36000061035156, |
| "completions/mean_terminated_length": 80.36000061035156, |
| "completions/min_length": 37.0, |
| "completions/min_terminated_length": 37.0, |
| "entropy": 0.22362540960311889, |
| "epoch": 0.3125, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01079811342060566, |
| "learning_rate": 5e-05, |
| "loss": -0.002070262096822262, |
| "num_tokens": 5902430.0, |
| "reward": 0.580761730670929, |
| "reward_std": 0.49483081698417664, |
| "rewards/correctness/mean": 0.6200000047683716, |
| "rewards/correctness/std": 0.4903143644332886, |
| "rewards/length_penalty/mean": -0.039238281548023224, |
| "rewards/length_penalty/std": 0.014868262223899364, |
| "sampling/importance_sampling_ratio/max": 2.2997214794158936, |
| "sampling/importance_sampling_ratio/mean": 0.9913575053215027, |
| "sampling/importance_sampling_ratio/min": 0.2111196517944336, |
| "sampling/sampling_logp_difference/max": 1.5553302764892578, |
| "sampling/sampling_logp_difference/mean": 0.034344468265771866, |
| "step": 115, |
| "step_time": 2.6515860850922763 |
| }, |
| { |
| "clip_ratio/high_max": 0.0019970620749518277, |
| "clip_ratio/high_mean": 0.0019970620749518277, |
| "clip_ratio/low_mean": 0.0002793296007439494, |
| "clip_ratio/low_min": 0.0002793296007439494, |
| "clip_ratio/region_mean": 0.00227639174554497, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 180.0, |
| "completions/max_terminated_length": 180.0, |
| "completions/mean_length": 78.41999816894531, |
| "completions/mean_terminated_length": 78.41999816894531, |
| "completions/min_length": 33.0, |
| "completions/min_terminated_length": 33.0, |
| "entropy": 0.24196649491786956, |
| "epoch": 0.31521739130434784, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.013271356001496315, |
| "learning_rate": 5e-05, |
| "loss": 0.0022566956467926502, |
| "num_tokens": 5909241.0, |
| "reward": 0.8017089366912842, |
| "reward_std": 0.37205037474632263, |
| "rewards/correctness/mean": 0.8399999737739563, |
| "rewards/correctness/std": 0.37032803893089294, |
| "rewards/length_penalty/mean": -0.03829101473093033, |
| "rewards/length_penalty/std": 0.015793830156326294, |
| "sampling/importance_sampling_ratio/max": 2.9569191932678223, |
| "sampling/importance_sampling_ratio/mean": 0.9928051829338074, |
| "sampling/importance_sampling_ratio/min": 0.2952917516231537, |
| "sampling/sampling_logp_difference/max": 1.2197914123535156, |
| "sampling/sampling_logp_difference/mean": 0.039991345256567, |
| "step": 116, |
| "step_time": 2.655353401787579 |
| }, |
| { |
| "clip_ratio/high_max": 0.0033249396830797194, |
| "clip_ratio/high_mean": 0.0033249396830797194, |
| "clip_ratio/low_mean": 0.0013393433764576912, |
| "clip_ratio/low_min": 0.0013393433764576912, |
| "clip_ratio/region_mean": 0.0046642830595374106, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 67.0, |
| "completions/max_terminated_length": 67.0, |
| "completions/mean_length": 43.599998474121094, |
| "completions/mean_terminated_length": 43.599998474121094, |
| "completions/min_length": 20.0, |
| "completions/min_terminated_length": 20.0, |
| "entropy": 0.2081553041934967, |
| "epoch": 0.3179347826086957, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.008443817496299744, |
| "learning_rate": 5e-05, |
| "loss": 0.0012317131040617824, |
| "num_tokens": 5913601.0, |
| "reward": 0.7587109208106995, |
| "reward_std": 0.4203566610813141, |
| "rewards/correctness/mean": 0.7799999713897705, |
| "rewards/correctness/std": 0.4184519648551941, |
| "rewards/length_penalty/mean": -0.02128906175494194, |
| "rewards/length_penalty/std": 0.005932000931352377, |
| "sampling/importance_sampling_ratio/max": 1.8893288373947144, |
| "sampling/importance_sampling_ratio/mean": 0.9916374087333679, |
| "sampling/importance_sampling_ratio/min": 0.27941209077835083, |
| "sampling/sampling_logp_difference/max": 1.2750675678253174, |
| "sampling/sampling_logp_difference/mean": 0.033489543944597244, |
| "step": 117, |
| "step_time": 1.8823761746753007 |
| }, |
| { |
| "clip_ratio/high_max": 0.0005688481265679002, |
| "clip_ratio/high_mean": 0.0005688481265679002, |
| "clip_ratio/low_mean": 0.0012103871908038855, |
| "clip_ratio/low_min": 0.0012103871908038855, |
| "clip_ratio/region_mean": 0.0017792353173717856, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 142.0, |
| "completions/max_terminated_length": 142.0, |
| "completions/mean_length": 72.47999572753906, |
| "completions/mean_terminated_length": 72.47999572753906, |
| "completions/min_length": 22.0, |
| "completions/min_terminated_length": 22.0, |
| "entropy": 0.27200756371021273, |
| "epoch": 0.32065217391304346, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.009959971532225609, |
| "learning_rate": 5e-05, |
| "loss": 0.001126274699345231, |
| "num_tokens": 5920905.0, |
| "reward": 0.6646093726158142, |
| "reward_std": 0.4720146656036377, |
| "rewards/correctness/mean": 0.699999988079071, |
| "rewards/correctness/std": 0.4629100263118744, |
| "rewards/length_penalty/mean": -0.03539062663912773, |
| "rewards/length_penalty/std": 0.015659291297197342, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9858464598655701, |
| "sampling/importance_sampling_ratio/min": 0.17056792974472046, |
| "sampling/sampling_logp_difference/max": 1.7686216831207275, |
| "sampling/sampling_logp_difference/mean": 0.04525027051568031, |
| "step": 118, |
| "step_time": 2.567681823624298 |
| }, |
| { |
| "clip_ratio/high_max": 0.0012298159534111619, |
| "clip_ratio/high_mean": 0.0012298159534111619, |
| "clip_ratio/low_mean": 0.0004323530476540327, |
| "clip_ratio/low_min": 0.0004323530476540327, |
| "clip_ratio/region_mean": 0.0016621690359897912, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1335.0, |
| "completions/max_terminated_length": 1335.0, |
| "completions/mean_length": 139.77999877929688, |
| "completions/mean_terminated_length": 139.77999877929688, |
| "completions/min_length": 15.0, |
| "completions/min_terminated_length": 15.0, |
| "entropy": 0.3178446352481842, |
| "epoch": 0.3233695652173913, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012043810449540615, |
| "learning_rate": 5e-05, |
| "loss": 0.011804413050413132, |
| "num_tokens": 5931404.0, |
| "reward": 0.4517480432987213, |
| "reward_std": 0.5475705862045288, |
| "rewards/correctness/mean": 0.5199999809265137, |
| "rewards/correctness/std": 0.5046720504760742, |
| "rewards/length_penalty/mean": -0.06825195252895355, |
| "rewards/length_penalty/std": 0.0961272343993187, |
| "sampling/importance_sampling_ratio/max": 2.9376678466796875, |
| "sampling/importance_sampling_ratio/mean": 0.9891913533210754, |
| "sampling/importance_sampling_ratio/min": 0.1745537668466568, |
| "sampling/sampling_logp_difference/max": 1.7455224990844727, |
| "sampling/sampling_logp_difference/mean": 0.03469213470816612, |
| "step": 119, |
| "step_time": 13.277865092735738 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007105602882802487, |
| "clip_ratio/high_mean": 0.0007105602882802487, |
| "clip_ratio/low_mean": 0.0009997313376516105, |
| "clip_ratio/low_min": 0.0009997313376516105, |
| "clip_ratio/region_mean": 0.001710291625931859, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 113.0, |
| "completions/max_terminated_length": 113.0, |
| "completions/mean_length": 57.2599983215332, |
| "completions/mean_terminated_length": 57.2599983215332, |
| "completions/min_length": 26.0, |
| "completions/min_terminated_length": 26.0, |
| "entropy": 0.28369104862213135, |
| "epoch": 0.32608695652173914, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.014030136168003082, |
| "learning_rate": 5e-05, |
| "loss": 0.0005326682003214955, |
| "num_tokens": 5937117.0, |
| "reward": 0.6920409798622131, |
| "reward_std": 0.45576512813568115, |
| "rewards/correctness/mean": 0.7200000286102295, |
| "rewards/correctness/std": 0.4535573422908783, |
| "rewards/length_penalty/mean": -0.027958983555436134, |
| "rewards/length_penalty/std": 0.011268665082752705, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9907137155532837, |
| "sampling/importance_sampling_ratio/min": 0.3402984142303467, |
| "sampling/sampling_logp_difference/max": 1.1216139793395996, |
| "sampling/sampling_logp_difference/mean": 0.04103627800941467, |
| "step": 120, |
| "step_time": 2.1914469429757446 |
| }, |
| { |
| "clip_ratio/high_max": 0.002339954860508442, |
| "clip_ratio/high_mean": 0.002339954860508442, |
| "clip_ratio/low_mean": 0.002092923759482801, |
| "clip_ratio/low_min": 0.002092923759482801, |
| "clip_ratio/region_mean": 0.004432878619991243, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 258.0, |
| "completions/max_terminated_length": 258.0, |
| "completions/mean_length": 77.43999481201172, |
| "completions/mean_terminated_length": 77.43999481201172, |
| "completions/min_length": 12.0, |
| "completions/min_terminated_length": 12.0, |
| "entropy": 0.4044592082500458, |
| "epoch": 0.328804347826087, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.013229206204414368, |
| "learning_rate": 5e-05, |
| "loss": 0.007325820624828339, |
| "num_tokens": 5944329.0, |
| "reward": 0.5021874904632568, |
| "reward_std": 0.5180118083953857, |
| "rewards/correctness/mean": 0.5400000214576721, |
| "rewards/correctness/std": 0.5034574270248413, |
| "rewards/length_penalty/mean": -0.037812501192092896, |
| "rewards/length_penalty/std": 0.029638368636369705, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9860542416572571, |
| "sampling/importance_sampling_ratio/min": 0.10667832940816879, |
| "sampling/sampling_logp_difference/max": 2.2379372119903564, |
| "sampling/sampling_logp_difference/mean": 0.04855887591838837, |
| "step": 121, |
| "step_time": 3.5320951712783426 |
| }, |
| { |
| "clip_ratio/high_max": 0.0020871169865131376, |
| "clip_ratio/high_mean": 0.0020871169865131376, |
| "clip_ratio/low_mean": 0.0006550624966621399, |
| "clip_ratio/low_min": 0.0006550624966621399, |
| "clip_ratio/region_mean": 0.0027421795064583422, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 342.0, |
| "completions/max_terminated_length": 342.0, |
| "completions/mean_length": 62.63999938964844, |
| "completions/mean_terminated_length": 62.63999938964844, |
| "completions/min_length": 19.0, |
| "completions/min_terminated_length": 19.0, |
| "entropy": 0.3629956305027008, |
| "epoch": 0.33152173913043476, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.011344057507812977, |
| "learning_rate": 5e-05, |
| "loss": 0.003215065225958824, |
| "num_tokens": 5950581.0, |
| "reward": 0.4694140553474426, |
| "reward_std": 0.5132778882980347, |
| "rewards/correctness/mean": 0.5, |
| "rewards/correctness/std": 0.5050762891769409, |
| "rewards/length_penalty/mean": -0.030585937201976776, |
| "rewards/length_penalty/std": 0.023655664175748825, |
| "sampling/importance_sampling_ratio/max": 2.0439770221710205, |
| "sampling/importance_sampling_ratio/mean": 0.9834688901901245, |
| "sampling/importance_sampling_ratio/min": 0.33990684151649475, |
| "sampling/sampling_logp_difference/max": 1.0790836811065674, |
| "sampling/sampling_logp_difference/mean": 0.043290577828884125, |
| "step": 122, |
| "step_time": 3.9696634637657553 |
| }, |
| { |
| "clip_ratio/high_max": 0.0021428524632938206, |
| "clip_ratio/high_mean": 0.0021428524632938206, |
| "clip_ratio/low_mean": 0.0003522538812831044, |
| "clip_ratio/low_min": 0.0003522538812831044, |
| "clip_ratio/region_mean": 0.0024951063678599894, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1630.0, |
| "completions/max_terminated_length": 1630.0, |
| "completions/mean_length": 105.41999816894531, |
| "completions/mean_terminated_length": 105.41999816894531, |
| "completions/min_length": 32.0, |
| "completions/min_terminated_length": 32.0, |
| "entropy": 0.25871089696884153, |
| "epoch": 0.3342391304347826, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.019554821774363518, |
| "learning_rate": 5e-05, |
| "loss": 0.02193872630596161, |
| "num_tokens": 5959092.0, |
| "reward": 0.34852537512779236, |
| "reward_std": 0.5240065455436707, |
| "rewards/correctness/mean": 0.4000000059604645, |
| "rewards/correctness/std": 0.4948716461658478, |
| "rewards/length_penalty/mean": -0.05147460848093033, |
| "rewards/length_penalty/std": 0.1117917150259018, |
| "sampling/importance_sampling_ratio/max": 2.471583604812622, |
| "sampling/importance_sampling_ratio/mean": 0.9906110167503357, |
| "sampling/importance_sampling_ratio/min": 0.1870785504579544, |
| "sampling/sampling_logp_difference/max": 1.6762266159057617, |
| "sampling/sampling_logp_difference/mean": 0.029479123651981354, |
| "step": 123, |
| "step_time": 15.582083855988458 |
| }, |
| { |
| "clip_ratio/high_max": 0.00041841003112494943, |
| "clip_ratio/high_mean": 0.00041841003112494943, |
| "clip_ratio/low_mean": 0.00041841003112494943, |
| "clip_ratio/low_min": 0.00041841003112494943, |
| "clip_ratio/region_mean": 0.0008368200622498989, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 103.0, |
| "completions/max_terminated_length": 103.0, |
| "completions/mean_length": 42.70000076293945, |
| "completions/mean_terminated_length": 42.70000076293945, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 0.18871377408504486, |
| "epoch": 0.33695652173913043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012075711973011494, |
| "learning_rate": 5e-05, |
| "loss": 0.003170022275298834, |
| "num_tokens": 5963477.0, |
| "reward": 0.5391503572463989, |
| "reward_std": 0.5063897967338562, |
| "rewards/correctness/mean": 0.5600000023841858, |
| "rewards/correctness/std": 0.5014265179634094, |
| "rewards/length_penalty/mean": -0.02084960974752903, |
| "rewards/length_penalty/std": 0.011277487501502037, |
| "sampling/importance_sampling_ratio/max": 2.757899284362793, |
| "sampling/importance_sampling_ratio/mean": 0.9949593544006348, |
| "sampling/importance_sampling_ratio/min": 0.26883986592292786, |
| "sampling/sampling_logp_difference/max": 1.3136394023895264, |
| "sampling/sampling_logp_difference/mean": 0.035366520285606384, |
| "step": 124, |
| "step_time": 2.537050310987979 |
| }, |
| { |
| "clip_ratio/high_max": 0.0037846322637051346, |
| "clip_ratio/high_mean": 0.0037846322637051346, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0037846322637051346, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 89.0, |
| "completions/max_terminated_length": 89.0, |
| "completions/mean_length": 43.02000045776367, |
| "completions/mean_terminated_length": 43.02000045776367, |
| "completions/min_length": 16.0, |
| "completions/min_terminated_length": 16.0, |
| "entropy": 0.2748114734888077, |
| "epoch": 0.33967391304347827, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.010747011750936508, |
| "learning_rate": 5e-05, |
| "loss": -1.1701034964062274e-05, |
| "num_tokens": 5968108.0, |
| "reward": 0.47899413108825684, |
| "reward_std": 0.5066565275192261, |
| "rewards/correctness/mean": 0.5, |
| "rewards/correctness/std": 0.5050762891769409, |
| "rewards/length_penalty/mean": -0.021005859598517418, |
| "rewards/length_penalty/std": 0.008610362187027931, |
| "sampling/importance_sampling_ratio/max": 1.9138907194137573, |
| "sampling/importance_sampling_ratio/mean": 0.990419864654541, |
| "sampling/importance_sampling_ratio/min": 0.3262421786785126, |
| "sampling/sampling_logp_difference/max": 1.1201152801513672, |
| "sampling/sampling_logp_difference/mean": 0.04402812942862511, |
| "step": 125, |
| "step_time": 1.9398821538779885 |
| }, |
| { |
| "clip_ratio/high_max": 0.0012025600532069801, |
| "clip_ratio/high_mean": 0.0012025600532069801, |
| "clip_ratio/low_mean": 0.0009474263759329915, |
| "clip_ratio/low_min": 0.0009474263759329915, |
| "clip_ratio/region_mean": 0.002149986429139972, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 247.0, |
| "completions/max_terminated_length": 247.0, |
| "completions/mean_length": 61.03999710083008, |
| "completions/mean_terminated_length": 61.03999710083008, |
| "completions/min_length": 17.0, |
| "completions/min_terminated_length": 17.0, |
| "entropy": 0.3201991945505142, |
| "epoch": 0.3423913043478261, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.013153769075870514, |
| "learning_rate": 5e-05, |
| "loss": 0.008743176236748695, |
| "num_tokens": 5976130.0, |
| "reward": 0.47019529342651367, |
| "reward_std": 0.5109017491340637, |
| "rewards/correctness/mean": 0.5, |
| "rewards/correctness/std": 0.5050762891769409, |
| "rewards/length_penalty/mean": -0.029804687947034836, |
| "rewards/length_penalty/std": 0.024028556421399117, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9863394498825073, |
| "sampling/importance_sampling_ratio/min": 0.2564975321292877, |
| "sampling/sampling_logp_difference/max": 1.3606362342834473, |
| "sampling/sampling_logp_difference/mean": 0.03647936135530472, |
| "step": 126, |
| "step_time": 3.403859922895208 |
| }, |
| { |
| "clip_ratio/high_max": 0.0022899978794157505, |
| "clip_ratio/high_mean": 0.0022899978794157505, |
| "clip_ratio/low_mean": 0.0016236664494499564, |
| "clip_ratio/low_min": 0.0016236664494499564, |
| "clip_ratio/region_mean": 0.003913664352148771, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 155.0, |
| "completions/max_terminated_length": 155.0, |
| "completions/mean_length": 53.21999740600586, |
| "completions/mean_terminated_length": 53.21999740600586, |
| "completions/min_length": 16.0, |
| "completions/min_terminated_length": 16.0, |
| "entropy": 0.2477138340473175, |
| "epoch": 0.3451086956521739, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.014389632269740105, |
| "learning_rate": 5e-05, |
| "loss": -0.0011505823349580169, |
| "num_tokens": 5981421.0, |
| "reward": 0.4540136754512787, |
| "reward_std": 0.5024483799934387, |
| "rewards/correctness/mean": 0.47999998927116394, |
| "rewards/correctness/std": 0.5046720504760742, |
| "rewards/length_penalty/mean": -0.025986328721046448, |
| "rewards/length_penalty/std": 0.0136498948559165, |
| "sampling/importance_sampling_ratio/max": 2.8656787872314453, |
| "sampling/importance_sampling_ratio/mean": 0.9904687404632568, |
| "sampling/importance_sampling_ratio/min": 0.3013729751110077, |
| "sampling/sampling_logp_difference/max": 1.199406623840332, |
| "sampling/sampling_logp_difference/mean": 0.03252456709742546, |
| "step": 127, |
| "step_time": 2.412539087701589 |
| }, |
| { |
| "clip_ratio/high_max": 0.0012546269223093986, |
| "clip_ratio/high_mean": 0.0012546269223093986, |
| "clip_ratio/low_mean": 0.00281849279999733, |
| "clip_ratio/low_min": 0.00281849279999733, |
| "clip_ratio/region_mean": 0.004073119722306728, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 211.0, |
| "completions/max_terminated_length": 211.0, |
| "completions/mean_length": 36.040000915527344, |
| "completions/mean_terminated_length": 36.040000915527344, |
| "completions/min_length": 14.0, |
| "completions/min_terminated_length": 14.0, |
| "entropy": 0.2539794921875, |
| "epoch": 0.34782608695652173, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.01464737206697464, |
| "learning_rate": 5e-05, |
| "loss": 0.002901690546423197, |
| "num_tokens": 5985543.0, |
| "reward": 0.662402331829071, |
| "reward_std": 0.4712248742580414, |
| "rewards/correctness/mean": 0.6800000071525574, |
| "rewards/correctness/std": 0.4712120592594147, |
| "rewards/length_penalty/mean": -0.017597656697034836, |
| "rewards/length_penalty/std": 0.013573279604315758, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9928796887397766, |
| "sampling/importance_sampling_ratio/min": 0.210645854473114, |
| "sampling/sampling_logp_difference/max": 1.5575768947601318, |
| "sampling/sampling_logp_difference/mean": 0.04199233278632164, |
| "step": 128, |
| "step_time": 2.5844002419617027 |
| }, |
| { |
| "clip_ratio/high_max": 0.002605846803635359, |
| "clip_ratio/high_mean": 0.002605846803635359, |
| "clip_ratio/low_mean": 0.0013825902715325356, |
| "clip_ratio/low_min": 0.0013825902715325356, |
| "clip_ratio/region_mean": 0.0039884370751678945, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 150.0, |
| "completions/max_terminated_length": 150.0, |
| "completions/mean_length": 45.81999969482422, |
| "completions/mean_terminated_length": 45.81999969482422, |
| "completions/min_length": 12.0, |
| "completions/min_terminated_length": 12.0, |
| "entropy": 0.29810507595539093, |
| "epoch": 0.35054347826086957, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.011710134334862232, |
| "learning_rate": 5e-05, |
| "loss": 0.003014075104147196, |
| "num_tokens": 5990894.0, |
| "reward": 0.39762693643569946, |
| "reward_std": 0.5041302442550659, |
| "rewards/correctness/mean": 0.41999998688697815, |
| "rewards/correctness/std": 0.49856939911842346, |
| "rewards/length_penalty/mean": -0.022373046725988388, |
| "rewards/length_penalty/std": 0.013797881081700325, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9921415448188782, |
| "sampling/importance_sampling_ratio/min": 0.32757124304771423, |
| "sampling/sampling_logp_difference/max": 2.4522218704223633, |
| "sampling/sampling_logp_difference/mean": 0.042723968625068665, |
| "step": 129, |
| "step_time": 2.299919289071113 |
| }, |
| { |
| "clip_ratio/high_max": 0.00097189387306571, |
| "clip_ratio/high_mean": 0.00097189387306571, |
| "clip_ratio/low_mean": 0.0014031045138835907, |
| "clip_ratio/low_min": 0.0014031045138835907, |
| "clip_ratio/region_mean": 0.002374998386949301, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 188.0, |
| "completions/max_terminated_length": 188.0, |
| "completions/mean_length": 41.63999938964844, |
| "completions/mean_terminated_length": 41.63999938964844, |
| "completions/min_length": 17.0, |
| "completions/min_terminated_length": 17.0, |
| "entropy": 0.2866079300642014, |
| "epoch": 0.3532608695652174, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.014074258506298065, |
| "learning_rate": 5e-05, |
| "loss": 0.0013535830657929182, |
| "num_tokens": 5995746.0, |
| "reward": 0.6996679306030273, |
| "reward_std": 0.45589467883110046, |
| "rewards/correctness/mean": 0.7200000286102295, |
| "rewards/correctness/std": 0.4535573720932007, |
| "rewards/length_penalty/mean": -0.020332030951976776, |
| "rewards/length_penalty/std": 0.01232099998742342, |
| "sampling/importance_sampling_ratio/max": 2.545606851577759, |
| "sampling/importance_sampling_ratio/mean": 0.9906572103500366, |
| "sampling/importance_sampling_ratio/min": 0.1368415504693985, |
| "sampling/sampling_logp_difference/max": 1.988931655883789, |
| "sampling/sampling_logp_difference/mean": 0.04350081458687782, |
| "step": 130, |
| "step_time": 2.440223409794271 |
| }, |
| { |
| "clip_ratio/high_max": 0.0022780835861340165, |
| "clip_ratio/high_mean": 0.0022780835861340165, |
| "clip_ratio/low_mean": 0.0007551305228844285, |
| "clip_ratio/low_min": 0.0007551305228844285, |
| "clip_ratio/region_mean": 0.003033214085735381, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 184.0, |
| "completions/max_terminated_length": 184.0, |
| "completions/mean_length": 48.5, |
| "completions/mean_terminated_length": 48.5, |
| "completions/min_length": 12.0, |
| "completions/min_terminated_length": 12.0, |
| "entropy": 0.2793427348136902, |
| "epoch": 0.35597826086956524, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.010720326565206051, |
| "learning_rate": 5e-05, |
| "loss": 0.0011166753247380257, |
| "num_tokens": 6001551.0, |
| "reward": 0.4363183379173279, |
| "reward_std": 0.5099425315856934, |
| "rewards/correctness/mean": 0.46000000834465027, |
| "rewards/correctness/std": 0.5034573674201965, |
| "rewards/length_penalty/mean": -0.023681640625, |
| "rewards/length_penalty/std": 0.016856711357831955, |
| "sampling/importance_sampling_ratio/max": 2.0086348056793213, |
| "sampling/importance_sampling_ratio/mean": 0.9880565404891968, |
| "sampling/importance_sampling_ratio/min": 0.1856611669063568, |
| "sampling/sampling_logp_difference/max": 1.6838319301605225, |
| "sampling/sampling_logp_difference/mean": 0.03877510875463486, |
| "step": 131, |
| "step_time": 2.758909401949495 |
| }, |
| { |
| "clip_ratio/high_max": 0.0014819981530308723, |
| "clip_ratio/high_mean": 0.0014819981530308723, |
| "clip_ratio/low_mean": 0.0006134969182312489, |
| "clip_ratio/low_min": 0.0006134969182312489, |
| "clip_ratio/region_mean": 0.0020954950712621214, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 96.0, |
| "completions/max_terminated_length": 96.0, |
| "completions/mean_length": 37.15999984741211, |
| "completions/mean_terminated_length": 37.15999984741211, |
| "completions/min_length": 14.0, |
| "completions/min_terminated_length": 14.0, |
| "entropy": 0.22479414641857148, |
| "epoch": 0.358695652173913, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.00991136860102415, |
| "learning_rate": 5e-05, |
| "loss": 0.0004250574274919927, |
| "num_tokens": 6005239.0, |
| "reward": 0.44185546040534973, |
| "reward_std": 0.5011681318283081, |
| "rewards/correctness/mean": 0.46000000834465027, |
| "rewards/correctness/std": 0.5034573674201965, |
| "rewards/length_penalty/mean": -0.018144531175494194, |
| "rewards/length_penalty/std": 0.010862166061997414, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.991742730140686, |
| "sampling/importance_sampling_ratio/min": 0.3487538695335388, |
| "sampling/sampling_logp_difference/max": 1.4278502464294434, |
| "sampling/sampling_logp_difference/mean": 0.03862390294671059, |
| "step": 132, |
| "step_time": 1.9299116518814117 |
| }, |
| { |
| "clip_ratio/high_max": 0.001252327300608158, |
| "clip_ratio/high_mean": 0.001252327300608158, |
| "clip_ratio/low_mean": 0.0018417619168758393, |
| "clip_ratio/low_min": 0.0018417619168758393, |
| "clip_ratio/region_mean": 0.0030940892174839973, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 88.0, |
| "completions/max_terminated_length": 88.0, |
| "completions/mean_length": 33.15999984741211, |
| "completions/mean_terminated_length": 33.15999984741211, |
| "completions/min_length": 18.0, |
| "completions/min_terminated_length": 18.0, |
| "entropy": 0.2817714989185333, |
| "epoch": 0.36141304347826086, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.008105136454105377, |
| "learning_rate": 5e-05, |
| "loss": -0.0009989547543227673, |
| "num_tokens": 6009587.0, |
| "reward": 0.42380857467651367, |
| "reward_std": 0.5010993480682373, |
| "rewards/correctness/mean": 0.4399999976158142, |
| "rewards/correctness/std": 0.5014265179634094, |
| "rewards/length_penalty/mean": -0.016191406175494194, |
| "rewards/length_penalty/std": 0.007762114051729441, |
| "sampling/importance_sampling_ratio/max": 1.943861961364746, |
| "sampling/importance_sampling_ratio/mean": 0.9887738227844238, |
| "sampling/importance_sampling_ratio/min": 0.2915809452533722, |
| "sampling/sampling_logp_difference/max": 1.2324376106262207, |
| "sampling/sampling_logp_difference/mean": 0.039197519421577454, |
| "step": 133, |
| "step_time": 1.915332199074328 |
| }, |
| { |
| "clip_ratio/high_max": 0.0008230452425777912, |
| "clip_ratio/high_mean": 0.0008230452425777912, |
| "clip_ratio/low_mean": 0.0007968127727508544, |
| "clip_ratio/low_min": 0.0007968127727508544, |
| "clip_ratio/region_mean": 0.0016198580153286457, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 58.0, |
| "completions/max_terminated_length": 58.0, |
| "completions/mean_length": 25.079999923706055, |
| "completions/mean_terminated_length": 25.079999923706055, |
| "completions/min_length": 15.0, |
| "completions/min_terminated_length": 15.0, |
| "entropy": 0.19418410062789918, |
| "epoch": 0.3641304347826087, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.010841830633580685, |
| "learning_rate": 5e-05, |
| "loss": -0.0005958230467513204, |
| "num_tokens": 6013531.0, |
| "reward": 0.5877538919448853, |
| "reward_std": 0.4935136139392853, |
| "rewards/correctness/mean": 0.6000000238418579, |
| "rewards/correctness/std": 0.4948716461658478, |
| "rewards/length_penalty/mean": -0.012246093712747097, |
| "rewards/length_penalty/std": 0.004991067107766867, |
| "sampling/importance_sampling_ratio/max": 2.5555429458618164, |
| "sampling/importance_sampling_ratio/mean": 0.9979280829429626, |
| "sampling/importance_sampling_ratio/min": 0.10886480659246445, |
| "sampling/sampling_logp_difference/max": 2.217648506164551, |
| "sampling/sampling_logp_difference/mean": 0.035169076174497604, |
| "step": 134, |
| "step_time": 1.825435406761244 |
| }, |
| { |
| "clip_ratio/high_max": 0.0028490145690739156, |
| "clip_ratio/high_mean": 0.0028490145690739156, |
| "clip_ratio/low_mean": 0.001599618257023394, |
| "clip_ratio/low_min": 0.001599618257023394, |
| "clip_ratio/region_mean": 0.004448632849380374, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 250.0, |
| "completions/max_terminated_length": 250.0, |
| "completions/mean_length": 33.68000030517578, |
| "completions/mean_terminated_length": 33.68000030517578, |
| "completions/min_length": 18.0, |
| "completions/min_terminated_length": 18.0, |
| "entropy": 0.2851863861083984, |
| "epoch": 0.36684782608695654, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.008571211248636246, |
| "learning_rate": 5e-05, |
| "loss": 0.002348761772736907, |
| "num_tokens": 6019395.0, |
| "reward": 0.12355468422174454, |
| "reward_std": 0.35180458426475525, |
| "rewards/correctness/mean": 0.14000000059604645, |
| "rewards/correctness/std": 0.3505098223686218, |
| "rewards/length_penalty/mean": -0.016445312649011612, |
| "rewards/length_penalty/std": 0.017781900241971016, |
| "sampling/importance_sampling_ratio/max": 2.548715114593506, |
| "sampling/importance_sampling_ratio/mean": 0.9878785610198975, |
| "sampling/importance_sampling_ratio/min": 0.14688512682914734, |
| "sampling/sampling_logp_difference/max": 1.9181044101715088, |
| "sampling/sampling_logp_difference/mean": 0.04530085623264313, |
| "step": 135, |
| "step_time": 3.565745212836191 |
| }, |
| { |
| "clip_ratio/high_max": 0.002418273827061057, |
| "clip_ratio/high_mean": 0.002418273827061057, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.002418273827061057, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 65.0, |
| "completions/max_terminated_length": 65.0, |
| "completions/mean_length": 33.36000061035156, |
| "completions/mean_terminated_length": 33.36000061035156, |
| "completions/min_length": 13.0, |
| "completions/min_terminated_length": 13.0, |
| "entropy": 0.2012420564889908, |
| "epoch": 0.3695652173913043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012487429194152355, |
| "learning_rate": 5e-05, |
| "loss": 0.0007071903673931956, |
| "num_tokens": 6023143.0, |
| "reward": 0.6437109112739563, |
| "reward_std": 0.47799545526504517, |
| "rewards/correctness/mean": 0.6600000262260437, |
| "rewards/correctness/std": 0.47851812839508057, |
| "rewards/length_penalty/mean": -0.016289062798023224, |
| "rewards/length_penalty/std": 0.006368517410010099, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.99516761302948, |
| "sampling/importance_sampling_ratio/min": 0.3091590404510498, |
| "sampling/sampling_logp_difference/max": 1.1738994121551514, |
| "sampling/sampling_logp_difference/mean": 0.035100940614938736, |
| "step": 136, |
| "step_time": 1.8060238698963076 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006153846159577369, |
| "clip_ratio/high_mean": 0.0006153846159577369, |
| "clip_ratio/low_mean": 0.0006153846159577369, |
| "clip_ratio/low_min": 0.0006153846159577369, |
| "clip_ratio/region_mean": 0.0012307692319154739, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 70.0, |
| "completions/max_terminated_length": 70.0, |
| "completions/mean_length": 41.459999084472656, |
| "completions/mean_terminated_length": 41.459999084472656, |
| "completions/min_length": 16.0, |
| "completions/min_terminated_length": 16.0, |
| "entropy": 0.17081710696220398, |
| "epoch": 0.37228260869565216, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.00821436196565628, |
| "learning_rate": 5e-05, |
| "loss": 0.0015817350940778852, |
| "num_tokens": 6028076.0, |
| "reward": 0.5397558212280273, |
| "reward_std": 0.5067362785339355, |
| "rewards/correctness/mean": 0.5600000023841858, |
| "rewards/correctness/std": 0.5014265179634094, |
| "rewards/length_penalty/mean": -0.020244140177965164, |
| "rewards/length_penalty/std": 0.009693717584013939, |
| "sampling/importance_sampling_ratio/max": 1.7419631481170654, |
| "sampling/importance_sampling_ratio/mean": 0.9927988648414612, |
| "sampling/importance_sampling_ratio/min": 0.35504549741744995, |
| "sampling/sampling_logp_difference/max": 1.0355093479156494, |
| "sampling/sampling_logp_difference/mean": 0.02509792521595955, |
| "step": 137, |
| "step_time": 1.8471284741535783 |
| }, |
| { |
| "clip_ratio/high_max": 0.0036086251493543386, |
| "clip_ratio/high_mean": 0.0036086251493543386, |
| "clip_ratio/low_mean": 0.0004056795034557581, |
| "clip_ratio/low_min": 0.0004056795034557581, |
| "clip_ratio/region_mean": 0.004014304652810097, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 188.0, |
| "completions/max_terminated_length": 188.0, |
| "completions/mean_length": 38.63999938964844, |
| "completions/mean_terminated_length": 38.63999938964844, |
| "completions/min_length": 18.0, |
| "completions/min_terminated_length": 18.0, |
| "entropy": 0.2256270557641983, |
| "epoch": 0.375, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.008576368913054466, |
| "learning_rate": 5e-05, |
| "loss": 0.003299054689705372, |
| "num_tokens": 6033038.0, |
| "reward": 0.5211328268051147, |
| "reward_std": 0.5057582855224609, |
| "rewards/correctness/mean": 0.5400000214576721, |
| "rewards/correctness/std": 0.5034574270248413, |
| "rewards/length_penalty/mean": -0.018867187201976776, |
| "rewards/length_penalty/std": 0.013564602471888065, |
| "sampling/importance_sampling_ratio/max": 1.6007344722747803, |
| "sampling/importance_sampling_ratio/mean": 0.985950231552124, |
| "sampling/importance_sampling_ratio/min": 0.07781390100717545, |
| "sampling/sampling_logp_difference/max": 2.5534350872039795, |
| "sampling/sampling_logp_difference/mean": 0.03910902887582779, |
| "step": 138, |
| "step_time": 2.515745670069009 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.00024968788493424655, |
| "clip_ratio/low_min": 0.00024968788493424655, |
| "clip_ratio/region_mean": 0.00024968788493424655, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 445.0, |
| "completions/max_terminated_length": 445.0, |
| "completions/mean_length": 37.70000076293945, |
| "completions/mean_terminated_length": 37.70000076293945, |
| "completions/min_length": 16.0, |
| "completions/min_terminated_length": 16.0, |
| "entropy": 0.2630132257938385, |
| "epoch": 0.37771739130434784, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.011270148679614067, |
| "learning_rate": 5e-05, |
| "loss": 0.0028373789973556995, |
| "num_tokens": 6037703.0, |
| "reward": 0.18159179389476776, |
| "reward_std": 0.4050198197364807, |
| "rewards/correctness/mean": 0.20000000298023224, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.01840820349752903, |
| "rewards/length_penalty/std": 0.030904652550816536, |
| "sampling/importance_sampling_ratio/max": 1.4473143815994263, |
| "sampling/importance_sampling_ratio/mean": 0.9855800271034241, |
| "sampling/importance_sampling_ratio/min": 0.45448043942451477, |
| "sampling/sampling_logp_difference/max": 0.7886004447937012, |
| "sampling/sampling_logp_difference/mean": 0.02931682951748371, |
| "step": 139, |
| "step_time": 4.710732836974785 |
| }, |
| { |
| "clip_ratio/high_max": 0.0039062283933162687, |
| "clip_ratio/high_mean": 0.0039062283933162687, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0039062283933162687, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 104.0, |
| "completions/max_terminated_length": 104.0, |
| "completions/mean_length": 34.459999084472656, |
| "completions/mean_terminated_length": 34.459999084472656, |
| "completions/min_length": 15.0, |
| "completions/min_terminated_length": 15.0, |
| "entropy": 0.28029139935970304, |
| "epoch": 0.3804347826086957, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.007939372211694717, |
| "learning_rate": 5e-05, |
| "loss": 0.001906823250465095, |
| "num_tokens": 6043066.0, |
| "reward": 0.48317381739616394, |
| "reward_std": 0.5078606009483337, |
| "rewards/correctness/mean": 0.5, |
| "rewards/correctness/std": 0.5050762891769409, |
| "rewards/length_penalty/mean": -0.016826171427965164, |
| "rewards/length_penalty/std": 0.008946106769144535, |
| "sampling/importance_sampling_ratio/max": 2.1778173446655273, |
| "sampling/importance_sampling_ratio/mean": 0.9844793081283569, |
| "sampling/importance_sampling_ratio/min": 0.2821851074695587, |
| "sampling/sampling_logp_difference/max": 1.2651920318603516, |
| "sampling/sampling_logp_difference/mean": 0.039756450802087784, |
| "step": 140, |
| "step_time": 2.190974553814158 |
| }, |
| { |
| "clip_ratio/high_max": 0.0021777387242764234, |
| "clip_ratio/high_mean": 0.0021777387242764234, |
| "clip_ratio/low_mean": 0.000526315812021494, |
| "clip_ratio/low_min": 0.000526315812021494, |
| "clip_ratio/region_mean": 0.00270405444316566, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 100.0, |
| "completions/max_terminated_length": 100.0, |
| "completions/mean_length": 44.41999816894531, |
| "completions/mean_terminated_length": 44.41999816894531, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 0.22660049200057983, |
| "epoch": 0.38315217391304346, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.010583036579191685, |
| "learning_rate": 5e-05, |
| "loss": 0.0017442009411752224, |
| "num_tokens": 6048017.0, |
| "reward": 0.4783105254173279, |
| "reward_std": 0.5048611164093018, |
| "rewards/correctness/mean": 0.5, |
| "rewards/correctness/std": 0.5050762891769409, |
| "rewards/length_penalty/mean": -0.02168945223093033, |
| "rewards/length_penalty/std": 0.011465261690318584, |
| "sampling/importance_sampling_ratio/max": 1.4931265115737915, |
| "sampling/importance_sampling_ratio/mean": 0.9867258071899414, |
| "sampling/importance_sampling_ratio/min": 0.09107369929552078, |
| "sampling/sampling_logp_difference/max": 2.3960862159729004, |
| "sampling/sampling_logp_difference/mean": 0.03287019953131676, |
| "step": 141, |
| "step_time": 2.040590178454295 |
| }, |
| { |
| "clip_ratio/high_max": 0.002919093193486333, |
| "clip_ratio/high_mean": 0.002919093193486333, |
| "clip_ratio/low_mean": 0.0005555555690079927, |
| "clip_ratio/low_min": 0.0005555555690079927, |
| "clip_ratio/region_mean": 0.003474648762494326, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 76.0, |
| "completions/max_terminated_length": 76.0, |
| "completions/mean_length": 29.8799991607666, |
| "completions/mean_terminated_length": 29.8799991607666, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 0.2625975370407104, |
| "epoch": 0.3858695652173913, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012548496946692467, |
| "learning_rate": 5e-05, |
| "loss": 0.0033927010372281075, |
| "num_tokens": 6052301.0, |
| "reward": 0.4254101514816284, |
| "reward_std": 0.5026375651359558, |
| "rewards/correctness/mean": 0.4399999976158142, |
| "rewards/correctness/std": 0.5014265179634094, |
| "rewards/length_penalty/mean": -0.014589843340218067, |
| "rewards/length_penalty/std": 0.008529257029294968, |
| "sampling/importance_sampling_ratio/max": 2.683242082595825, |
| "sampling/importance_sampling_ratio/mean": 0.9934856295585632, |
| "sampling/importance_sampling_ratio/min": 0.07049904763698578, |
| "sampling/sampling_logp_difference/max": 2.652156114578247, |
| "sampling/sampling_logp_difference/mean": 0.04649779945611954, |
| "step": 142, |
| "step_time": 1.7738669028040022 |
| }, |
| { |
| "clip_ratio/high_max": 0.0008064515888690949, |
| "clip_ratio/high_mean": 0.0008064515888690949, |
| "clip_ratio/low_mean": 0.00048076924867928027, |
| "clip_ratio/low_min": 0.00048076924867928027, |
| "clip_ratio/region_mean": 0.0012872208375483752, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 92.0, |
| "completions/max_terminated_length": 92.0, |
| "completions/mean_length": 35.23999786376953, |
| "completions/mean_terminated_length": 35.23999786376953, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 0.18691827356815338, |
| "epoch": 0.38858695652173914, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.009847329929471016, |
| "learning_rate": 5e-05, |
| "loss": 0.0008599873399361968, |
| "num_tokens": 6056403.0, |
| "reward": 0.44279295206069946, |
| "reward_std": 0.5066936612129211, |
| "rewards/correctness/mean": 0.46000000834465027, |
| "rewards/correctness/std": 0.5034574270248413, |
| "rewards/length_penalty/mean": -0.017207032069563866, |
| "rewards/length_penalty/std": 0.01047922857105732, |
| "sampling/importance_sampling_ratio/max": 2.7992422580718994, |
| "sampling/importance_sampling_ratio/mean": 0.9902405142784119, |
| "sampling/importance_sampling_ratio/min": 0.17679700255393982, |
| "sampling/sampling_logp_difference/max": 1.732753038406372, |
| "sampling/sampling_logp_difference/mean": 0.034317996352910995, |
| "step": 143, |
| "step_time": 2.020391206955537 |
| }, |
| { |
| "clip_ratio/high_max": 0.000803212821483612, |
| "clip_ratio/high_mean": 0.000803212821483612, |
| "clip_ratio/low_mean": 0.0014033276587724687, |
| "clip_ratio/low_min": 0.0014033276587724687, |
| "clip_ratio/region_mean": 0.0022065404802560805, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 92.0, |
| "completions/max_terminated_length": 92.0, |
| "completions/mean_length": 28.059999465942383, |
| "completions/mean_terminated_length": 28.059999465942383, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 0.1982872635126114, |
| "epoch": 0.391304347826087, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012820910662412643, |
| "learning_rate": 5e-05, |
| "loss": 0.0011645274935290217, |
| "num_tokens": 6060856.0, |
| "reward": 0.24629881978034973, |
| "reward_std": 0.444132924079895, |
| "rewards/correctness/mean": 0.25999999046325684, |
| "rewards/correctness/std": 0.44308751821517944, |
| "rewards/length_penalty/mean": -0.013701171614229679, |
| "rewards/length_penalty/std": 0.008656531572341919, |
| "sampling/importance_sampling_ratio/max": 2.300398588180542, |
| "sampling/importance_sampling_ratio/mean": 0.9945727586746216, |
| "sampling/importance_sampling_ratio/min": 0.28775298595428467, |
| "sampling/sampling_logp_difference/max": 1.2456529140472412, |
| "sampling/sampling_logp_difference/mean": 0.03747813031077385, |
| "step": 144, |
| "step_time": 1.8891234768088907 |
| }, |
| { |
| "clip_ratio/high_max": 0.0028685005381703377, |
| "clip_ratio/high_mean": 0.0028685005381703377, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0028685005381703377, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 35.0, |
| "completions/max_terminated_length": 35.0, |
| "completions/mean_length": 20.760000228881836, |
| "completions/mean_terminated_length": 20.760000228881836, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 0.11449335515499115, |
| "epoch": 0.39402173913043476, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.022868067026138306, |
| "learning_rate": 5e-05, |
| "loss": 0.001642507268115878, |
| "num_tokens": 6064224.0, |
| "reward": 0.46986326575279236, |
| "reward_std": 0.5068603754043579, |
| "rewards/correctness/mean": 0.47999998927116394, |
| "rewards/correctness/std": 0.5046720504760742, |
| "rewards/length_penalty/mean": -0.010136718861758709, |
| "rewards/length_penalty/std": 0.004064429551362991, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9941112399101257, |
| "sampling/importance_sampling_ratio/min": 0.18363036215305328, |
| "sampling/sampling_logp_difference/max": 1.6948304176330566, |
| "sampling/sampling_logp_difference/mean": 0.0298063512891531, |
| "step": 145, |
| "step_time": 1.5340910970699042 |
| }, |
| { |
| "clip_ratio/high_max": 0.005621016025543213, |
| "clip_ratio/high_mean": 0.005621016025543213, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.005621016025543213, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 37.0, |
| "completions/max_terminated_length": 37.0, |
| "completions/mean_length": 14.34000015258789, |
| "completions/mean_terminated_length": 14.34000015258789, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 0.15054220706224442, |
| "epoch": 0.3967391304347826, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012355758808553219, |
| "learning_rate": 5e-05, |
| "loss": 0.0012912503443658352, |
| "num_tokens": 6067881.0, |
| "reward": 0.5529980063438416, |
| "reward_std": 0.5011084079742432, |
| "rewards/correctness/mean": 0.5600000023841858, |
| "rewards/correctness/std": 0.5014265179634094, |
| "rewards/length_penalty/mean": -0.007001953199505806, |
| "rewards/length_penalty/std": 0.0030906975734978914, |
| "sampling/importance_sampling_ratio/max": 2.6878232955932617, |
| "sampling/importance_sampling_ratio/mean": 0.9853367805480957, |
| "sampling/importance_sampling_ratio/min": 0.16923557221889496, |
| "sampling/sampling_logp_difference/max": 1.7764636278152466, |
| "sampling/sampling_logp_difference/mean": 0.05857422575354576, |
| "step": 146, |
| "step_time": 1.4935915581882 |
| }, |
| { |
| "clip_ratio/high_max": 0.004120059683918953, |
| "clip_ratio/high_mean": 0.004120059683918953, |
| "clip_ratio/low_mean": 0.001342281885445118, |
| "clip_ratio/low_min": 0.001342281885445118, |
| "clip_ratio/region_mean": 0.0054623415693640705, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 29.0, |
| "completions/max_terminated_length": 29.0, |
| "completions/mean_length": 14.15999984741211, |
| "completions/mean_terminated_length": 14.15999984741211, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 0.09627678245306015, |
| "epoch": 0.39945652173913043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.048468802124261856, |
| "learning_rate": 5e-05, |
| "loss": 0.0005410637240856886, |
| "num_tokens": 6070889.0, |
| "reward": 0.33308592438697815, |
| "reward_std": 0.4778694212436676, |
| "rewards/correctness/mean": 0.3400000035762787, |
| "rewards/correctness/std": 0.4785180985927582, |
| "rewards/length_penalty/mean": -0.006914062425494194, |
| "rewards/length_penalty/std": 0.0017294606659561396, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9871498346328735, |
| "sampling/importance_sampling_ratio/min": 0.21893155574798584, |
| "sampling/sampling_logp_difference/max": 1.5690662860870361, |
| "sampling/sampling_logp_difference/mean": 0.03849093243479729, |
| "step": 147, |
| "step_time": 1.9382816620636731 |
| }, |
| { |
| "clip_ratio/high_max": 0.001785714365541935, |
| "clip_ratio/high_mean": 0.001785714365541935, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.001785714365541935, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14.0, |
| "completions/max_terminated_length": 14.0, |
| "completions/mean_length": 11.219999313354492, |
| "completions/mean_terminated_length": 11.219999313354492, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 0.09468620419502258, |
| "epoch": 0.40217391304347827, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.013497934676706791, |
| "learning_rate": 5e-05, |
| "loss": 0.0001785533968359232, |
| "num_tokens": 6074360.0, |
| "reward": 0.1945214867591858, |
| "reward_std": 0.4041154980659485, |
| "rewards/correctness/mean": 0.20000000298023224, |
| "rewards/correctness/std": 0.4040610194206238, |
| "rewards/length_penalty/mean": -0.0054785157553851604, |
| "rewards/length_penalty/std": 0.0004101894737686962, |
| "sampling/importance_sampling_ratio/max": 1.3334838151931763, |
| "sampling/importance_sampling_ratio/mean": 0.9918619394302368, |
| "sampling/importance_sampling_ratio/min": 0.1652003973722458, |
| "sampling/sampling_logp_difference/max": 1.800595998764038, |
| "sampling/sampling_logp_difference/mean": 0.02222876064479351, |
| "step": 148, |
| "step_time": 1.357269709231332 |
| }, |
| { |
| "clip_ratio/high_max": 0.0011363636702299118, |
| "clip_ratio/high_mean": 0.0011363636702299118, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0011363636702299118, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 36.0, |
| "completions/max_terminated_length": 36.0, |
| "completions/mean_length": 14.019999504089355, |
| "completions/mean_terminated_length": 14.019999504089355, |
| "completions/min_length": 11.0, |
| "completions/min_terminated_length": 11.0, |
| "entropy": 0.1659936398267746, |
| "epoch": 0.4048913043478261, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.008294458501040936, |
| "learning_rate": 5e-05, |
| "loss": 0.0007371972315013409, |
| "num_tokens": 6078031.0, |
| "reward": 0.1731542944908142, |
| "reward_std": 0.38879117369651794, |
| "rewards/correctness/mean": 0.18000000715255737, |
| "rewards/correctness/std": 0.3880879282951355, |
| "rewards/length_penalty/mean": -0.006845703348517418, |
| "rewards/length_penalty/std": 0.002153346547856927, |
| "sampling/importance_sampling_ratio/max": 1.6676440238952637, |
| "sampling/importance_sampling_ratio/mean": 0.9931860566139221, |
| "sampling/importance_sampling_ratio/min": 0.24514369666576385, |
| "sampling/sampling_logp_difference/max": 1.4059107303619385, |
| "sampling/sampling_logp_difference/mean": 0.02321179211139679, |
| "step": 149, |
| "step_time": 1.5234273611567914 |
| }, |
| { |
| "clip_ratio/high_max": 0.0015037594363093375, |
| "clip_ratio/high_mean": 0.0015037594363093375, |
| "clip_ratio/low_mean": 0.0015037594363093375, |
| "clip_ratio/low_min": 0.0015037594363093375, |
| "clip_ratio/region_mean": 0.003007518872618675, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 30.0, |
| "completions/max_terminated_length": 30.0, |
| "completions/mean_length": 11.5, |
| "completions/mean_terminated_length": 11.5, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 0.030891793966293334, |
| "epoch": 0.4076086956521739, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.012591220438480377, |
| "learning_rate": 5e-05, |
| "loss": -0.00041141733527183533, |
| "num_tokens": 6080366.0, |
| "reward": 0.5543847680091858, |
| "reward_std": 0.5016669034957886, |
| "rewards/correctness/mean": 0.5600000023841858, |
| "rewards/correctness/std": 0.5014265179634094, |
| "rewards/length_penalty/mean": -0.005615234375, |
| "rewards/length_penalty/std": 0.0014098342508077621, |
| "sampling/importance_sampling_ratio/max": 1.1196001768112183, |
| "sampling/importance_sampling_ratio/mean": 0.9954295754432678, |
| "sampling/importance_sampling_ratio/min": 0.48469099402427673, |
| "sampling/sampling_logp_difference/max": 0.7242437601089478, |
| "sampling/sampling_logp_difference/mean": 0.009189309552311897, |
| "step": 150, |
| "step_time": 1.4204991972073913 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 200, |
| "num_input_tokens_seen": 6080366, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 10, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
