| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9945822640433419, |
| "eval_steps": 500, |
| "global_step": 109, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 982.0, |
| "completions/max_terminated_length": 982.0, |
| "completions/mean_length": 520.5390625, |
| "completions/mean_terminated_length": 520.5390625, |
| "completions/min_length": 302.0, |
| "completions/min_terminated_length": 302.0, |
| "entropy": 0.114654541015625, |
| "epoch": 0.009124607927003137, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0243235994130373, |
| "kl": 0.00023239850997924805, |
| "learning_rate": 2e-05, |
| "loss": -0.0022, |
| "num_tokens": 408495.0, |
| "reward": 0.963108241558075, |
| "reward_std": 0.2168295979499817, |
| "rewards/cross_solution_unittest_reward/mean": 0.48155416548252106, |
| "rewards/cross_solution_unittest_reward/std": 0.22725073248147964, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 1 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1246.0, |
| "completions/max_terminated_length": 1246.0, |
| "completions/mean_length": 493.14324951171875, |
| "completions/mean_terminated_length": 493.14324951171875, |
| "completions/min_length": 200.0, |
| "completions/min_terminated_length": 200.0, |
| "entropy": 0.1129150390625, |
| "epoch": 0.018249215854006275, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02810436487197876, |
| "kl": 0.00023671984672546387, |
| "learning_rate": 1.9995921928281893e-05, |
| "loss": -0.004, |
| "num_tokens": 819398.0, |
| "reward": 0.7592691779136658, |
| "reward_std": 0.20917575061321259, |
| "rewards/cross_solution_unittest_reward/mean": 0.3796345815062523, |
| "rewards/cross_solution_unittest_reward/std": 0.2533634677529335, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 2 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1210.0, |
| "completions/max_terminated_length": 1210.0, |
| "completions/mean_length": 584.3802490234375, |
| "completions/mean_terminated_length": 584.3802490234375, |
| "completions/min_length": 240.0, |
| "completions/min_terminated_length": 240.0, |
| "entropy": 0.09423828125, |
| "epoch": 0.02737382378100941, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.024342378601431847, |
| "kl": 0.00020760297775268555, |
| "learning_rate": 1.9983691039261358e-05, |
| "loss": -0.0025, |
| "num_tokens": 1264568.0, |
| "reward": 0.8243939280509949, |
| "reward_std": 0.19790497422218323, |
| "rewards/cross_solution_unittest_reward/mean": 0.41219695657491684, |
| "rewards/cross_solution_unittest_reward/std": 0.20426170527935028, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 3 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 902.0, |
| "completions/mean_length": 510.72137451171875, |
| "completions/mean_terminated_length": 494.6308898925781, |
| "completions/min_length": 293.0, |
| "completions/min_terminated_length": 293.0, |
| "entropy": 0.12127685546875, |
| "epoch": 0.03649843170801255, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02658909745514393, |
| "kl": 0.00026023387908935547, |
| "learning_rate": 1.9963317308626916e-05, |
| "loss": -0.0008, |
| "num_tokens": 1707901.0, |
| "reward": 0.6146944165229797, |
| "reward_std": 0.20338332653045654, |
| "rewards/cross_solution_unittest_reward/mean": 0.30734721571207047, |
| "rewards/cross_solution_unittest_reward/std": 0.1820686012506485, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 4 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1694.0, |
| "completions/mean_length": 495.9921875, |
| "completions/mean_terminated_length": 487.92950439453125, |
| "completions/min_length": 228.0, |
| "completions/min_terminated_length": 228.0, |
| "entropy": 0.1229400634765625, |
| "epoch": 0.04562303963501568, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03194279596209526, |
| "kl": 0.0002792775630950928, |
| "learning_rate": 1.99348173534855e-05, |
| "loss": 0.0193, |
| "num_tokens": 2092986.0, |
| "reward": 0.8980245590209961, |
| "reward_std": 0.2468685805797577, |
| "rewards/cross_solution_unittest_reward/mean": 0.44901224970817566, |
| "rewards/cross_solution_unittest_reward/std": 0.2793443500995636, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 5 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1340.0, |
| "completions/mean_length": 548.34375, |
| "completions/mean_terminated_length": 532.4502563476562, |
| "completions/min_length": 297.0, |
| "completions/min_terminated_length": 297.0, |
| "entropy": 0.1123809814453125, |
| "epoch": 0.05474764756201882, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.024234920740127563, |
| "kl": 0.0002339184284210205, |
| "learning_rate": 1.989821441880933e-05, |
| "loss": 0.0085, |
| "num_tokens": 2517798.0, |
| "reward": 0.8548812866210938, |
| "reward_std": 0.16416706144809723, |
| "rewards/cross_solution_unittest_reward/mean": 0.42744066566228867, |
| "rewards/cross_solution_unittest_reward/std": 0.24477335810661316, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 6 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 3107.0, |
| "completions/max_terminated_length": 3107.0, |
| "completions/mean_length": 531.8958740234375, |
| "completions/mean_terminated_length": 531.8958740234375, |
| "completions/min_length": 232.0, |
| "completions/min_terminated_length": 232.0, |
| "entropy": 0.1142120361328125, |
| "epoch": 0.06387225548902195, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.025173544883728027, |
| "kl": 0.00025781989097595215, |
| "learning_rate": 1.9853538358476933e-05, |
| "loss": -0.0049, |
| "num_tokens": 2962550.0, |
| "reward": 0.8058516383171082, |
| "reward_std": 0.26573851704597473, |
| "rewards/cross_solution_unittest_reward/mean": 0.4029258117079735, |
| "rewards/cross_solution_unittest_reward/std": 0.2175045683979988, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 7 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1891.0, |
| "completions/max_terminated_length": 1891.0, |
| "completions/mean_length": 522.03125, |
| "completions/mean_terminated_length": 522.03125, |
| "completions/min_length": 214.0, |
| "completions/min_terminated_length": 214.0, |
| "entropy": 0.1124114990234375, |
| "epoch": 0.0729968634160251, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.025463413447141647, |
| "kl": 0.00025027990341186523, |
| "learning_rate": 1.9800825610923937e-05, |
| "loss": 0.0018, |
| "num_tokens": 3417666.0, |
| "reward": 0.7429351210594177, |
| "reward_std": 0.21776002645492554, |
| "rewards/cross_solution_unittest_reward/mean": 0.37146755307912827, |
| "rewards/cross_solution_unittest_reward/std": 0.2737540230154991, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 8 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1064.0, |
| "completions/mean_length": 495.703125, |
| "completions/mean_terminated_length": 487.6396789550781, |
| "completions/min_length": 250.0, |
| "completions/min_terminated_length": 250.0, |
| "entropy": 0.12152099609375, |
| "epoch": 0.08212147134302823, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02767878770828247, |
| "kl": 0.0002829134464263916, |
| "learning_rate": 1.9740119169423337e-05, |
| "loss": -0.0138, |
| "num_tokens": 3811456.0, |
| "reward": 0.8020304441452026, |
| "reward_std": 0.24267295002937317, |
| "rewards/cross_solution_unittest_reward/mean": 0.40101519227027893, |
| "rewards/cross_solution_unittest_reward/std": 0.21552012115716934, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 9 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 994.0, |
| "completions/max_terminated_length": 994.0, |
| "completions/mean_length": 502.54168701171875, |
| "completions/mean_terminated_length": 502.54168701171875, |
| "completions/min_length": 250.0, |
| "completions/min_terminated_length": 250.0, |
| "entropy": 0.1065216064453125, |
| "epoch": 0.09124607927003137, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029218332841992378, |
| "kl": 0.0002897977828979492, |
| "learning_rate": 1.9671468547019575e-05, |
| "loss": 0.0047, |
| "num_tokens": 4254672.0, |
| "reward": 0.8215359449386597, |
| "reward_std": 0.20994165539741516, |
| "rewards/cross_solution_unittest_reward/mean": 0.41076793521642685, |
| "rewards/cross_solution_unittest_reward/std": 0.2297622635960579, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 10 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1138.0, |
| "completions/max_terminated_length": 1138.0, |
| "completions/mean_length": 546.2109375, |
| "completions/mean_terminated_length": 546.2109375, |
| "completions/min_length": 264.0, |
| "completions/min_terminated_length": 264.0, |
| "entropy": 0.1016082763671875, |
| "epoch": 0.1003706871970345, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02588176354765892, |
| "kl": 0.00028312206268310547, |
| "learning_rate": 1.9594929736144978e-05, |
| "loss": 0.0049, |
| "num_tokens": 4680241.0, |
| "reward": 0.829417884349823, |
| "reward_std": 0.19322270154953003, |
| "rewards/cross_solution_unittest_reward/mean": 0.4147089347243309, |
| "rewards/cross_solution_unittest_reward/std": 0.20234351605176926, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 11 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 907.0, |
| "completions/mean_length": 506.40625, |
| "completions/mean_terminated_length": 490.293212890625, |
| "completions/min_length": 255.0, |
| "completions/min_terminated_length": 255.0, |
| "entropy": 0.1226959228515625, |
| "epoch": 0.10949529512403763, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03448358178138733, |
| "kl": 0.00037539005279541016, |
| "learning_rate": 1.9510565162951538e-05, |
| "loss": -0.0022, |
| "num_tokens": 5112749.0, |
| "reward": 0.6837499141693115, |
| "reward_std": 0.23883536458015442, |
| "rewards/cross_solution_unittest_reward/mean": 0.34187494963407516, |
| "rewards/cross_solution_unittest_reward/std": 0.24100807309150696, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 12 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1035.0, |
| "completions/max_terminated_length": 1035.0, |
| "completions/mean_length": 534.4818115234375, |
| "completions/mean_terminated_length": 534.4818115234375, |
| "completions/min_length": 247.0, |
| "completions/min_terminated_length": 247.0, |
| "entropy": 0.1075286865234375, |
| "epoch": 0.11861990305104078, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027723070234060287, |
| "kl": 0.0003420412540435791, |
| "learning_rate": 1.941844363639525e-05, |
| "loss": -0.0135, |
| "num_tokens": 5525534.0, |
| "reward": 0.8237327337265015, |
| "reward_std": 0.19265763461589813, |
| "rewards/cross_solution_unittest_reward/mean": 0.41186633706092834, |
| "rewards/cross_solution_unittest_reward/std": 0.24469294399023056, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 13 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 2351.0, |
| "completions/max_terminated_length": 2351.0, |
| "completions/mean_length": 615.5104370117188, |
| "completions/mean_terminated_length": 615.5104370117188, |
| "completions/min_length": 281.0, |
| "completions/min_terminated_length": 281.0, |
| "entropy": 0.1039581298828125, |
| "epoch": 0.1277445109780439, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.025678865611553192, |
| "kl": 0.00036969780921936035, |
| "learning_rate": 1.9318640292114526e-05, |
| "loss": -0.0176, |
| "num_tokens": 5993994.0, |
| "reward": 0.8750783205032349, |
| "reward_std": 0.16023778915405273, |
| "rewards/cross_solution_unittest_reward/mean": 0.43753916025161743, |
| "rewards/cross_solution_unittest_reward/std": 0.25390665233135223, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 14 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 826.0, |
| "completions/mean_length": 467.890625, |
| "completions/mean_terminated_length": 459.75457763671875, |
| "completions/min_length": 211.0, |
| "completions/min_terminated_length": 211.0, |
| "entropy": 0.122222900390625, |
| "epoch": 0.13686911890504705, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030205531045794487, |
| "kl": 0.00045168399810791016, |
| "learning_rate": 1.92112365311485e-05, |
| "loss": 0.0082, |
| "num_tokens": 6339752.0, |
| "reward": 0.8696179389953613, |
| "reward_std": 0.2131870836019516, |
| "rewards/cross_solution_unittest_reward/mean": 0.43480896949768066, |
| "rewards/cross_solution_unittest_reward/std": 0.2413550242781639, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 15 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 3274.0, |
| "completions/max_terminated_length": 3274.0, |
| "completions/mean_length": 536.6796875, |
| "completions/mean_terminated_length": 536.6796875, |
| "completions/min_length": 279.0, |
| "completions/min_terminated_length": 279.0, |
| "entropy": 0.1075592041015625, |
| "epoch": 0.1459937268320502, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027554774656891823, |
| "kl": 0.0004489421844482422, |
| "learning_rate": 1.9096319953545186e-05, |
| "loss": 0.001, |
| "num_tokens": 6744981.0, |
| "reward": 0.8156441450119019, |
| "reward_std": 0.22033701837062836, |
| "rewards/cross_solution_unittest_reward/mean": 0.40782201290130615, |
| "rewards/cross_solution_unittest_reward/std": 0.19903528690338135, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 16 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1317.0, |
| "completions/mean_length": 522.8880615234375, |
| "completions/mean_terminated_length": 514.8955688476562, |
| "completions/min_length": 253.0, |
| "completions/min_terminated_length": 253.0, |
| "entropy": 0.1123504638671875, |
| "epoch": 0.15511833475905332, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027508998289704323, |
| "kl": 0.0005013346672058105, |
| "learning_rate": 1.8973984286913584e-05, |
| "loss": 0.0082, |
| "num_tokens": 7159690.0, |
| "reward": 0.9808885455131531, |
| "reward_std": 0.23347638547420502, |
| "rewards/cross_solution_unittest_reward/mean": 0.49044425785541534, |
| "rewards/cross_solution_unittest_reward/std": 0.28279364109039307, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 17 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1142.0, |
| "completions/max_terminated_length": 1142.0, |
| "completions/mean_length": 522.3984375, |
| "completions/mean_terminated_length": 522.3984375, |
| "completions/min_length": 242.0, |
| "completions/min_terminated_length": 242.0, |
| "entropy": 0.1051788330078125, |
| "epoch": 0.16424294268605646, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027137286961078644, |
| "kl": 0.0005218386650085449, |
| "learning_rate": 1.8844329309978146e-05, |
| "loss": -0.0001, |
| "num_tokens": 7590731.0, |
| "reward": 0.8454278707504272, |
| "reward_std": 0.19084712862968445, |
| "rewards/cross_solution_unittest_reward/mean": 0.42271392047405243, |
| "rewards/cross_solution_unittest_reward/std": 0.22437505424022675, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 18 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 928.0, |
| "completions/mean_length": 507.03125, |
| "completions/mean_terminated_length": 498.9974060058594, |
| "completions/min_length": 227.0, |
| "completions/min_terminated_length": 227.0, |
| "entropy": 0.1159210205078125, |
| "epoch": 0.17336755061305958, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03020695596933365, |
| "kl": 0.0006084442138671875, |
| "learning_rate": 1.8707460771197773e-05, |
| "loss": 0.006, |
| "num_tokens": 8003919.0, |
| "reward": 0.7663086652755737, |
| "reward_std": 0.24537038803100586, |
| "rewards/cross_solution_unittest_reward/mean": 0.38315432518720627, |
| "rewards/cross_solution_unittest_reward/std": 0.2557295635342598, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 19 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1019.0, |
| "completions/max_terminated_length": 1019.0, |
| "completions/mean_length": 534.296875, |
| "completions/mean_terminated_length": 534.296875, |
| "completions/min_length": 272.0, |
| "completions/min_terminated_length": 272.0, |
| "entropy": 0.1091766357421875, |
| "epoch": 0.18249215854006273, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03100455552339554, |
| "kl": 0.0006772875785827637, |
| "learning_rate": 1.856349030251589e-05, |
| "loss": -0.0103, |
| "num_tokens": 8459457.0, |
| "reward": 0.7904119491577148, |
| "reward_std": 0.23983600735664368, |
| "rewards/cross_solution_unittest_reward/mean": 0.39520594477653503, |
| "rewards/cross_solution_unittest_reward/std": 0.23401054739952087, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 20 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 846.0, |
| "completions/max_terminated_length": 846.0, |
| "completions/mean_length": 485.5390625, |
| "completions/mean_terminated_length": 485.5390625, |
| "completions/min_length": 236.0, |
| "completions/min_terminated_length": 236.0, |
| "entropy": 0.10430908203125, |
| "epoch": 0.19161676646706588, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032105084508657455, |
| "kl": 0.0006991028785705566, |
| "learning_rate": 1.8412535328311813e-05, |
| "loss": -0.0052, |
| "num_tokens": 8844608.0, |
| "reward": 0.8521513938903809, |
| "reward_std": 0.21744942665100098, |
| "rewards/cross_solution_unittest_reward/mean": 0.4260757490992546, |
| "rewards/cross_solution_unittest_reward/std": 0.21132270246744156, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 21 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1558.0, |
| "completions/mean_length": 513.4479370117188, |
| "completions/mean_terminated_length": 505.4308166503906, |
| "completions/min_length": 242.0, |
| "completions/min_terminated_length": 242.0, |
| "entropy": 0.1094970703125, |
| "epoch": 0.200741374394069, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029629480093717575, |
| "kl": 0.0006964206695556641, |
| "learning_rate": 1.825471896962774e-05, |
| "loss": -0.0126, |
| "num_tokens": 9223148.0, |
| "reward": 0.8136462569236755, |
| "reward_std": 0.21329909563064575, |
| "rewards/cross_solution_unittest_reward/mean": 0.4068231135606766, |
| "rewards/cross_solution_unittest_reward/std": 0.24406656622886658, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 22 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1091.0, |
| "completions/mean_length": 513.3724365234375, |
| "completions/mean_terminated_length": 505.3551025390625, |
| "completions/min_length": 243.0, |
| "completions/min_terminated_length": 243.0, |
| "entropy": 0.1093292236328125, |
| "epoch": 0.20986598232107215, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028564205393195152, |
| "kl": 0.0008405447006225586, |
| "learning_rate": 1.8090169943749477e-05, |
| "loss": 0.0105, |
| "num_tokens": 9671827.0, |
| "reward": 0.9153262376785278, |
| "reward_std": 0.19782906770706177, |
| "rewards/cross_solution_unittest_reward/mean": 0.45766307413578033, |
| "rewards/cross_solution_unittest_reward/std": 0.23260264098644257, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 23 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 970.0, |
| "completions/max_terminated_length": 970.0, |
| "completions/mean_length": 499.43231201171875, |
| "completions/mean_terminated_length": 499.43231201171875, |
| "completions/min_length": 221.0, |
| "completions/min_terminated_length": 221.0, |
| "entropy": 0.1064300537109375, |
| "epoch": 0.21899059024807527, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02926481142640114, |
| "kl": 0.0008217096328735352, |
| "learning_rate": 1.7919022459222754e-05, |
| "loss": 0.0034, |
| "num_tokens": 10091281.0, |
| "reward": 0.7448365092277527, |
| "reward_std": 0.2386973798274994, |
| "rewards/cross_solution_unittest_reward/mean": 0.37241824716329575, |
| "rewards/cross_solution_unittest_reward/std": 0.2582271620631218, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 24 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1083.0, |
| "completions/mean_length": 496.18231201171875, |
| "completions/mean_terminated_length": 488.1201171875, |
| "completions/min_length": 278.0, |
| "completions/min_terminated_length": 278.0, |
| "entropy": 0.11749267578125, |
| "epoch": 0.22811519817507842, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03824177011847496, |
| "kl": 0.001001119613647461, |
| "learning_rate": 1.7741416106390828e-05, |
| "loss": 0.0011, |
| "num_tokens": 10495487.0, |
| "reward": 0.8449365496635437, |
| "reward_std": 0.20980952680110931, |
| "rewards/cross_solution_unittest_reward/mean": 0.42246825993061066, |
| "rewards/cross_solution_unittest_reward/std": 0.21189714968204498, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 25 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 848.0, |
| "completions/max_terminated_length": 848.0, |
| "completions/mean_length": 497.1458435058594, |
| "completions/mean_terminated_length": 497.1458435058594, |
| "completions/min_length": 214.0, |
| "completions/min_terminated_length": 214.0, |
| "entropy": 0.11029052734375, |
| "epoch": 0.23723980610208156, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028777848929166794, |
| "kl": 0.0008819103240966797, |
| "learning_rate": 1.7557495743542586e-05, |
| "loss": -0.0032, |
| "num_tokens": 10853559.0, |
| "reward": 0.7521219253540039, |
| "reward_std": 0.18529602885246277, |
| "rewards/cross_solution_unittest_reward/mean": 0.37606097012758255, |
| "rewards/cross_solution_unittest_reward/std": 0.24145866930484772, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 26 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 766.0, |
| "completions/max_terminated_length": 766.0, |
| "completions/mean_length": 468.05731201171875, |
| "completions/mean_terminated_length": 468.05731201171875, |
| "completions/min_length": 253.0, |
| "completions/min_terminated_length": 253.0, |
| "entropy": 0.1114501953125, |
| "epoch": 0.24636441402908468, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02834751084446907, |
| "kl": 0.0009758472442626953, |
| "learning_rate": 1.736741137876405e-05, |
| "loss": -0.0084, |
| "num_tokens": 11230181.0, |
| "reward": 0.8723708987236023, |
| "reward_std": 0.19247429072856903, |
| "rewards/cross_solution_unittest_reward/mean": 0.4361855015158653, |
| "rewards/cross_solution_unittest_reward/std": 0.21063780039548874, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 27 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1015.0, |
| "completions/mean_length": 507.0989685058594, |
| "completions/mean_terminated_length": 499.0652770996094, |
| "completions/min_length": 234.0, |
| "completions/min_terminated_length": 234.0, |
| "entropy": 0.1055908203125, |
| "epoch": 0.2554890219560878, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029102731496095657, |
| "kl": 0.0011304616928100586, |
| "learning_rate": 1.7171318047589637e-05, |
| "loss": -0.0081, |
| "num_tokens": 11663099.0, |
| "reward": 0.9932507276535034, |
| "reward_std": 0.18286040425300598, |
| "rewards/cross_solution_unittest_reward/mean": 0.4966253191232681, |
| "rewards/cross_solution_unittest_reward/std": 0.2659049928188324, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 28 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1139.0, |
| "completions/mean_length": 489.8515625, |
| "completions/mean_terminated_length": 481.7728576660156, |
| "completions/min_length": 275.0, |
| "completions/min_terminated_length": 275.0, |
| "entropy": 0.113128662109375, |
| "epoch": 0.26461362988309095, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030337095260620117, |
| "kl": 0.0015281438827514648, |
| "learning_rate": 1.696937568655294e-05, |
| "loss": 0.0098, |
| "num_tokens": 12059194.0, |
| "reward": 0.8777666091918945, |
| "reward_std": 0.2015705108642578, |
| "rewards/cross_solution_unittest_reward/mean": 0.43888331949710846, |
| "rewards/cross_solution_unittest_reward/std": 0.289621040225029, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 29 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 863.0, |
| "completions/max_terminated_length": 863.0, |
| "completions/mean_length": 518.9349365234375, |
| "completions/mean_terminated_length": 518.9349365234375, |
| "completions/min_length": 249.0, |
| "completions/min_terminated_length": 249.0, |
| "entropy": 0.10247802734375, |
| "epoch": 0.2737382378100941, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028208184987306595, |
| "kl": 0.001240849494934082, |
| "learning_rate": 1.6761749002740195e-05, |
| "loss": -0.0072, |
| "num_tokens": 12477345.0, |
| "reward": 0.7518314123153687, |
| "reward_std": 0.223491370677948, |
| "rewards/cross_solution_unittest_reward/mean": 0.37591565400362015, |
| "rewards/cross_solution_unittest_reward/std": 0.2666970565915108, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 30 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 766.0, |
| "completions/mean_length": 470.55731201171875, |
| "completions/mean_terminated_length": 462.4281921386719, |
| "completions/min_length": 248.0, |
| "completions/min_terminated_length": 248.0, |
| "entropy": 0.10906982421875, |
| "epoch": 0.28286284573709725, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03271934762597084, |
| "kl": 0.0014473199844360352, |
| "learning_rate": 1.6548607339452853e-05, |
| "loss": 0.0095, |
| "num_tokens": 12841471.0, |
| "reward": 0.9323596954345703, |
| "reward_std": 0.1906483769416809, |
| "rewards/cross_solution_unittest_reward/mean": 0.46617985516786575, |
| "rewards/cross_solution_unittest_reward/std": 0.21004272997379303, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 31 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 906.0, |
| "completions/mean_length": 483.4505310058594, |
| "completions/mean_terminated_length": 475.3551025390625, |
| "completions/min_length": 223.0, |
| "completions/min_terminated_length": 223.0, |
| "entropy": 0.105499267578125, |
| "epoch": 0.2919874536641004, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.045950278639793396, |
| "kl": 0.00164031982421875, |
| "learning_rate": 1.6330124538088705e-05, |
| "loss": 0.0251, |
| "num_tokens": 13295500.0, |
| "reward": 0.9082147479057312, |
| "reward_std": 0.3082047402858734, |
| "rewards/cross_solution_unittest_reward/mean": 0.4541073888540268, |
| "rewards/cross_solution_unittest_reward/std": 0.24801740050315857, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 32 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 778.0, |
| "completions/mean_length": 476.5130310058594, |
| "completions/mean_terminated_length": 468.39947509765625, |
| "completions/min_length": 273.0, |
| "completions/min_terminated_length": 273.0, |
| "entropy": 0.11260986328125, |
| "epoch": 0.3011120615911035, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03122815489768982, |
| "kl": 0.001780867576599121, |
| "learning_rate": 1.6106478796354382e-05, |
| "loss": 0.0068, |
| "num_tokens": 13676457.0, |
| "reward": 0.8194860816001892, |
| "reward_std": 0.1633000671863556, |
| "rewards/cross_solution_unittest_reward/mean": 0.4097430929541588, |
| "rewards/cross_solution_unittest_reward/std": 0.2260184958577156, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 33 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 794.0, |
| "completions/max_terminated_length": 794.0, |
| "completions/mean_length": 480.32293701171875, |
| "completions/mean_terminated_length": 480.32293701171875, |
| "completions/min_length": 266.0, |
| "completions/min_terminated_length": 266.0, |
| "entropy": 0.10528564453125, |
| "epoch": 0.31023666951810663, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03177294880151749, |
| "kl": 0.002082347869873047, |
| "learning_rate": 1.5877852522924733e-05, |
| "loss": -0.0068, |
| "num_tokens": 14100165.0, |
| "reward": 0.9499843120574951, |
| "reward_std": 0.18478095531463623, |
| "rewards/cross_solution_unittest_reward/mean": 0.47499214112758636, |
| "rewards/cross_solution_unittest_reward/std": 0.23216352611780167, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 34 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1548.0, |
| "completions/max_terminated_length": 1548.0, |
| "completions/mean_length": 490.5208435058594, |
| "completions/mean_terminated_length": 490.5208435058594, |
| "completions/min_length": 222.0, |
| "completions/min_terminated_length": 222.0, |
| "entropy": 0.109344482421875, |
| "epoch": 0.3193612774451098, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03219935670495033, |
| "kl": 0.001997709274291992, |
| "learning_rate": 1.5644432188667695e-05, |
| "loss": 0.0101, |
| "num_tokens": 14479845.0, |
| "reward": 0.8249267339706421, |
| "reward_std": 0.22756871581077576, |
| "rewards/cross_solution_unittest_reward/mean": 0.41246332228183746, |
| "rewards/cross_solution_unittest_reward/std": 0.249651201069355, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 35 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1074.0, |
| "completions/max_terminated_length": 1074.0, |
| "completions/mean_length": 508.5546875, |
| "completions/mean_terminated_length": 508.5546875, |
| "completions/min_length": 254.0, |
| "completions/min_terminated_length": 254.0, |
| "entropy": 0.110809326171875, |
| "epoch": 0.32848588537211293, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033212192356586456, |
| "kl": 0.0023381710052490234, |
| "learning_rate": 1.5406408174555978e-05, |
| "loss": -0.007, |
| "num_tokens": 14897138.0, |
| "reward": 0.7724038362503052, |
| "reward_std": 0.2212439626455307, |
| "rewards/cross_solution_unittest_reward/mean": 0.3862019032239914, |
| "rewards/cross_solution_unittest_reward/std": 0.2340112403035164, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 36 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 957.0, |
| "completions/max_terminated_length": 957.0, |
| "completions/mean_length": 499.2239685058594, |
| "completions/mean_terminated_length": 499.2239685058594, |
| "completions/min_length": 244.0, |
| "completions/min_terminated_length": 244.0, |
| "entropy": 0.0943603515625, |
| "epoch": 0.3376104932991161, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032685279846191406, |
| "kl": 0.0021796226501464844, |
| "learning_rate": 1.5163974616389621e-05, |
| "loss": -0.004, |
| "num_tokens": 15299728.0, |
| "reward": 0.8877171874046326, |
| "reward_std": 0.1671483963727951, |
| "rewards/cross_solution_unittest_reward/mean": 0.4438585937023163, |
| "rewards/cross_solution_unittest_reward/std": 0.2307201847434044, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 37 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1086.0, |
| "completions/max_terminated_length": 1086.0, |
| "completions/mean_length": 479.34375, |
| "completions/mean_terminated_length": 479.34375, |
| "completions/min_length": 190.0, |
| "completions/min_terminated_length": 190.0, |
| "entropy": 0.1061859130859375, |
| "epoch": 0.34673510122611917, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.031533192843198776, |
| "kl": 0.0025458335876464844, |
| "learning_rate": 1.491732924645604e-05, |
| "loss": 0.0057, |
| "num_tokens": 15681980.0, |
| "reward": 0.8841136693954468, |
| "reward_std": 0.18882207572460175, |
| "rewards/cross_solution_unittest_reward/mean": 0.442056804895401, |
| "rewards/cross_solution_unittest_reward/std": 0.23417016118764877, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 38 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 975.0, |
| "completions/mean_length": 462.5520935058594, |
| "completions/mean_terminated_length": 454.402099609375, |
| "completions/min_length": 202.0, |
| "completions/min_terminated_length": 202.0, |
| "entropy": 0.11846923828125, |
| "epoch": 0.3558597091531223, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.09556223452091217, |
| "kl": 0.0063495635986328125, |
| "learning_rate": 1.4666673232256738e-05, |
| "loss": 0.0078, |
| "num_tokens": 15989736.0, |
| "reward": 0.7538800239562988, |
| "reward_std": 0.2889000177383423, |
| "rewards/cross_solution_unittest_reward/mean": 0.3769400045275688, |
| "rewards/cross_solution_unittest_reward/std": 0.2265937775373459, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 39 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1821.0, |
| "completions/mean_length": 492.40625, |
| "completions/mean_terminated_length": 484.3341979980469, |
| "completions/min_length": 233.0, |
| "completions/min_terminated_length": 233.0, |
| "entropy": 0.1049957275390625, |
| "epoch": 0.36498431708012546, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.10226094722747803, |
| "kl": 0.006191730499267578, |
| "learning_rate": 1.4412211012432213e-05, |
| "loss": -0.0045, |
| "num_tokens": 16399484.0, |
| "reward": 0.7214807271957397, |
| "reward_std": 0.20936036109924316, |
| "rewards/cross_solution_unittest_reward/mean": 0.3607403114438057, |
| "rewards/cross_solution_unittest_reward/std": 0.286220021545887, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 40 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.953125, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1305.0, |
| "completions/mean_length": 521.59375, |
| "completions/mean_terminated_length": 497.4803161621094, |
| "completions/min_length": 272.0, |
| "completions/min_terminated_length": 272.0, |
| "entropy": 0.1065673828125, |
| "epoch": 0.3741089250071286, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03374212980270386, |
| "kl": 0.003753662109375, |
| "learning_rate": 1.4154150130018867e-05, |
| "loss": 0.0217, |
| "num_tokens": 16819632.0, |
| "reward": 0.6985855102539062, |
| "reward_std": 0.17247408628463745, |
| "rewards/cross_solution_unittest_reward/mean": 0.34929272532463074, |
| "rewards/cross_solution_unittest_reward/std": 0.21424289792776108, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 41 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 777.0, |
| "completions/mean_length": 498.4192810058594, |
| "completions/mean_terminated_length": 490.3629455566406, |
| "completions/min_length": 229.0, |
| "completions/min_terminated_length": 229.0, |
| "entropy": 0.0918121337890625, |
| "epoch": 0.38323353293413176, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0317884236574173, |
| "kl": 0.00311279296875, |
| "learning_rate": 1.3892701063173917e-05, |
| "loss": 0.0088, |
| "num_tokens": 17261449.0, |
| "reward": 0.8251139521598816, |
| "reward_std": 0.16213150322437286, |
| "rewards/cross_solution_unittest_reward/mean": 0.4125569462776184, |
| "rewards/cross_solution_unittest_reward/std": 0.23735850304365158, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 42 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 758.0, |
| "completions/max_terminated_length": 758.0, |
| "completions/mean_length": 464.19012451171875, |
| "completions/mean_terminated_length": 464.19012451171875, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 256.0, |
| "entropy": 0.103759765625, |
| "epoch": 0.39235814086113485, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033187512308359146, |
| "kl": 0.0038928985595703125, |
| "learning_rate": 1.362807705350641e-05, |
| "loss": 0.0018, |
| "num_tokens": 17594050.0, |
| "reward": 0.9705511927604675, |
| "reward_std": 0.15792036056518555, |
| "rewards/cross_solution_unittest_reward/mean": 0.48527561128139496, |
| "rewards/cross_solution_unittest_reward/std": 0.25453296303749084, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 43 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1812.0, |
| "completions/mean_length": 511.6640625, |
| "completions/mean_terminated_length": 503.6423034667969, |
| "completions/min_length": 238.0, |
| "completions/min_terminated_length": 238.0, |
| "entropy": 0.0930633544921875, |
| "epoch": 0.401482748788138, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02915688417851925, |
| "kl": 0.0028166770935058594, |
| "learning_rate": 1.3360493932154301e-05, |
| "loss": 0.0031, |
| "num_tokens": 18001433.0, |
| "reward": 0.6709164977073669, |
| "reward_std": 0.17882965505123138, |
| "rewards/cross_solution_unittest_reward/mean": 0.3354582153260708, |
| "rewards/cross_solution_unittest_reward/std": 0.20766809582710266, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 44 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 855.0, |
| "completions/mean_length": 456.7265625, |
| "completions/mean_terminated_length": 448.5613708496094, |
| "completions/min_length": 205.0, |
| "completions/min_terminated_length": 205.0, |
| "entropy": 0.1132659912109375, |
| "epoch": 0.41060735671514115, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.038488611578941345, |
| "kl": 0.004986763000488281, |
| "learning_rate": 1.3090169943749475e-05, |
| "loss": 0.0137, |
| "num_tokens": 18379368.0, |
| "reward": 0.9005228281021118, |
| "reward_std": 0.19103503227233887, |
| "rewards/cross_solution_unittest_reward/mean": 0.4502613842487335, |
| "rewards/cross_solution_unittest_reward/std": 0.25590047985315323, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 45 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 3564.0, |
| "completions/max_terminated_length": 3564.0, |
| "completions/mean_length": 498.2630310058594, |
| "completions/mean_terminated_length": 498.2630310058594, |
| "completions/min_length": 244.0, |
| "completions/min_terminated_length": 244.0, |
| "entropy": 0.098541259765625, |
| "epoch": 0.4197319646421443, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.034151654690504074, |
| "kl": 0.0035920143127441406, |
| "learning_rate": 1.2817325568414299e-05, |
| "loss": -0.0086, |
| "num_tokens": 18790933.0, |
| "reward": 0.7998638153076172, |
| "reward_std": 0.26316848397254944, |
| "rewards/cross_solution_unittest_reward/mean": 0.3999319151043892, |
| "rewards/cross_solution_unittest_reward/std": 0.2370620146393776, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 46 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 908.0, |
| "completions/max_terminated_length": 908.0, |
| "completions/mean_length": 479.18231201171875, |
| "completions/mean_terminated_length": 479.18231201171875, |
| "completions/min_length": 219.0, |
| "completions/min_terminated_length": 219.0, |
| "entropy": 0.11773681640625, |
| "epoch": 0.42885657256914744, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.037232693284749985, |
| "kl": 0.004299640655517578, |
| "learning_rate": 1.2542183341934873e-05, |
| "loss": -0.0027, |
| "num_tokens": 19200115.0, |
| "reward": 0.667292058467865, |
| "reward_std": 0.18406903743743896, |
| "rewards/cross_solution_unittest_reward/mean": 0.3336460068821907, |
| "rewards/cross_solution_unittest_reward/std": 0.25224795937538147, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 47 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 866.0, |
| "completions/max_terminated_length": 866.0, |
| "completions/mean_length": 480.40625, |
| "completions/mean_terminated_length": 480.40625, |
| "completions/min_length": 245.0, |
| "completions/min_terminated_length": 245.0, |
| "entropy": 0.1033172607421875, |
| "epoch": 0.43798118049615054, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0333259254693985, |
| "kl": 0.003824949264526367, |
| "learning_rate": 1.2264967674257647e-05, |
| "loss": -0.0009, |
| "num_tokens": 19586559.0, |
| "reward": 0.8440192937850952, |
| "reward_std": 0.23113533854484558, |
| "rewards/cross_solution_unittest_reward/mean": 0.4220096617937088, |
| "rewards/cross_solution_unittest_reward/std": 0.2280796319246292, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 48 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 998.0, |
| "completions/mean_length": 486.1458435058594, |
| "completions/mean_terminated_length": 478.05743408203125, |
| "completions/min_length": 267.0, |
| "completions/min_terminated_length": 267.0, |
| "entropy": 0.09979248046875, |
| "epoch": 0.4471057884231537, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03385407105088234, |
| "kl": 0.003530740737915039, |
| "learning_rate": 1.1985904666457455e-05, |
| "loss": 0.0061, |
| "num_tokens": 19995087.0, |
| "reward": 0.6481320261955261, |
| "reward_std": 0.26141294836997986, |
| "rewards/cross_solution_unittest_reward/mean": 0.3240659758448601, |
| "rewards/cross_solution_unittest_reward/std": 0.25914470851421356, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 49 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 966.0, |
| "completions/max_terminated_length": 966.0, |
| "completions/mean_length": 514.703125, |
| "completions/mean_terminated_length": 514.703125, |
| "completions/min_length": 251.0, |
| "completions/min_terminated_length": 251.0, |
| "entropy": 0.0972137451171875, |
| "epoch": 0.45623039635015683, |
| "frac_reward_zero_std": 0.0833333358168602, |
| "grad_norm": 0.03325730189681053, |
| "kl": 0.004256248474121094, |
| "learning_rate": 1.170522192632624e-05, |
| "loss": -0.0051, |
| "num_tokens": 20408813.0, |
| "reward": 0.8064446449279785, |
| "reward_std": 0.22871556878089905, |
| "rewards/cross_solution_unittest_reward/mean": 0.40322232991456985, |
| "rewards/cross_solution_unittest_reward/std": 0.28535734862089157, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 50 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 834.0, |
| "completions/mean_length": 510.9453125, |
| "completions/mean_terminated_length": 502.92169189453125, |
| "completions/min_length": 169.0, |
| "completions/min_terminated_length": 169.0, |
| "entropy": 0.0950927734375, |
| "epoch": 0.46535500427716, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03309718891978264, |
| "kl": 0.004229068756103516, |
| "learning_rate": 1.1423148382732854e-05, |
| "loss": 0.0142, |
| "num_tokens": 20829704.0, |
| "reward": 0.9345626831054688, |
| "reward_std": 0.21369287371635437, |
| "rewards/cross_solution_unittest_reward/mean": 0.46728135645389557, |
| "rewards/cross_solution_unittest_reward/std": 0.2026936635375023, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 51 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 760.0, |
| "completions/max_terminated_length": 760.0, |
| "completions/mean_length": 485.2786560058594, |
| "completions/mean_terminated_length": 485.2786560058594, |
| "completions/min_length": 208.0, |
| "completions/min_terminated_length": 208.0, |
| "entropy": 0.0979461669921875, |
| "epoch": 0.4744796122041631, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4690159559249878, |
| "kl": 0.013956546783447266, |
| "learning_rate": 1.1139914098905406e-05, |
| "loss": 0.0024, |
| "num_tokens": 21222691.0, |
| "reward": 0.9599242210388184, |
| "reward_std": 0.23365139961242676, |
| "rewards/cross_solution_unittest_reward/mean": 0.479962095618248, |
| "rewards/cross_solution_unittest_reward/std": 0.24594762176275253, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 52 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 891.0, |
| "completions/max_terminated_length": 891.0, |
| "completions/mean_length": 477.48699951171875, |
| "completions/mean_terminated_length": 477.48699951171875, |
| "completions/min_length": 233.0, |
| "completions/min_terminated_length": 233.0, |
| "entropy": 0.095062255859375, |
| "epoch": 0.4836042201311662, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03301261365413666, |
| "kl": 0.0033338069915771484, |
| "learning_rate": 1.08557500847884e-05, |
| "loss": 0.0043, |
| "num_tokens": 21619614.0, |
| "reward": 0.7612521052360535, |
| "reward_std": 0.18586371839046478, |
| "rewards/cross_solution_unittest_reward/mean": 0.38062604516744614, |
| "rewards/cross_solution_unittest_reward/std": 0.22070425003767014, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 53 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 992.0, |
| "completions/mean_length": 529.8698120117188, |
| "completions/mean_terminated_length": 521.8955688476562, |
| "completions/min_length": 272.0, |
| "completions/min_terminated_length": 272.0, |
| "entropy": 0.098876953125, |
| "epoch": 0.49272882805816937, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.037023600190877914, |
| "kl": 0.005710601806640625, |
| "learning_rate": 1.0570888108627682e-05, |
| "loss": -0.0049, |
| "num_tokens": 22079444.0, |
| "reward": 0.7765120267868042, |
| "reward_std": 0.25535768270492554, |
| "rewards/cross_solution_unittest_reward/mean": 0.3882560133934021, |
| "rewards/cross_solution_unittest_reward/std": 0.21029234677553177, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 54 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 968.0, |
| "completions/max_terminated_length": 968.0, |
| "completions/mean_length": 480.6302185058594, |
| "completions/mean_terminated_length": 480.6302185058594, |
| "completions/min_length": 233.0, |
| "completions/min_terminated_length": 233.0, |
| "entropy": 0.1016998291015625, |
| "epoch": 0.5018534359851725, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03228428214788437, |
| "kl": 0.003750324249267578, |
| "learning_rate": 1.0285560507936962e-05, |
| "loss": -0.0011, |
| "num_tokens": 22452846.0, |
| "reward": 0.7638306617736816, |
| "reward_std": 0.2091209590435028, |
| "rewards/cross_solution_unittest_reward/mean": 0.38191530853509903, |
| "rewards/cross_solution_unittest_reward/std": 0.21358844637870789, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 55 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 886.0, |
| "completions/max_terminated_length": 886.0, |
| "completions/mean_length": 487.8567810058594, |
| "completions/mean_terminated_length": 487.8567810058594, |
| "completions/min_length": 213.0, |
| "completions/min_terminated_length": 213.0, |
| "entropy": 0.1000823974609375, |
| "epoch": 0.5109780439121756, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032228607684373856, |
| "kl": 0.0042340755462646484, |
| "learning_rate": 1e-05, |
| "loss": 0.0103, |
| "num_tokens": 22825151.0, |
| "reward": 0.826278567314148, |
| "reward_std": 0.19069427251815796, |
| "rewards/cross_solution_unittest_reward/mean": 0.4131392240524292, |
| "rewards/cross_solution_unittest_reward/std": 0.21790383756160736, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 56 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1560.0, |
| "completions/max_terminated_length": 1560.0, |
| "completions/mean_length": 494.5520935058594, |
| "completions/mean_terminated_length": 494.5520935058594, |
| "completions/min_length": 277.0, |
| "completions/min_terminated_length": 277.0, |
| "entropy": 0.099639892578125, |
| "epoch": 0.5201026518391788, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.5714826583862305, |
| "kl": 0.06778192520141602, |
| "learning_rate": 9.71443949206304e-06, |
| "loss": 0.0054, |
| "num_tokens": 23267683.0, |
| "reward": 0.8670098185539246, |
| "reward_std": 0.18901364505290985, |
| "rewards/cross_solution_unittest_reward/mean": 0.4335048869252205, |
| "rewards/cross_solution_unittest_reward/std": 0.2112448662519455, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 57 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.953125, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 966.0, |
| "completions/mean_length": 494.9739685058594, |
| "completions/mean_terminated_length": 470.6509094238281, |
| "completions/min_length": 238.0, |
| "completions/min_terminated_length": 238.0, |
| "entropy": 0.1165924072265625, |
| "epoch": 0.5292272597661819, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03703828528523445, |
| "kl": 0.004994392395019531, |
| "learning_rate": 9.42911189137232e-06, |
| "loss": 0.0108, |
| "num_tokens": 23682969.0, |
| "reward": 0.8357396125793457, |
| "reward_std": 0.1958376169204712, |
| "rewards/cross_solution_unittest_reward/mean": 0.41786976903676987, |
| "rewards/cross_solution_unittest_reward/std": 0.28136271238327026, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 58 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 906.0, |
| "completions/max_terminated_length": 906.0, |
| "completions/mean_length": 464.0755310058594, |
| "completions/mean_terminated_length": 464.0755310058594, |
| "completions/min_length": 228.0, |
| "completions/min_terminated_length": 228.0, |
| "entropy": 0.1112823486328125, |
| "epoch": 0.5383518676931851, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03699680417776108, |
| "kl": 0.004660606384277344, |
| "learning_rate": 9.144249915211605e-06, |
| "loss": -0.0151, |
| "num_tokens": 24082606.0, |
| "reward": 0.8085569739341736, |
| "reward_std": 0.1914159059524536, |
| "rewards/cross_solution_unittest_reward/mean": 0.4042784869670868, |
| "rewards/cross_solution_unittest_reward/std": 0.2565099969506264, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 59 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1127.0, |
| "completions/max_terminated_length": 1127.0, |
| "completions/mean_length": 488.7890625, |
| "completions/mean_terminated_length": 488.7890625, |
| "completions/min_length": 258.0, |
| "completions/min_terminated_length": 258.0, |
| "entropy": 0.093719482421875, |
| "epoch": 0.5474764756201882, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030238306149840355, |
| "kl": 0.004515171051025391, |
| "learning_rate": 8.860085901094595e-06, |
| "loss": 0.003, |
| "num_tokens": 24514333.0, |
| "reward": 0.8814453482627869, |
| "reward_std": 0.23693448305130005, |
| "rewards/cross_solution_unittest_reward/mean": 0.44072266668081284, |
| "rewards/cross_solution_unittest_reward/std": 0.22619559615850449, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 60 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 984.0, |
| "completions/max_terminated_length": 984.0, |
| "completions/mean_length": 526.2005615234375, |
| "completions/mean_terminated_length": 526.2005615234375, |
| "completions/min_length": 270.0, |
| "completions/min_terminated_length": 270.0, |
| "entropy": 0.0959625244140625, |
| "epoch": 0.5566010835471913, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03212675452232361, |
| "kl": 0.0042629241943359375, |
| "learning_rate": 8.576851617267151e-06, |
| "loss": -0.0098, |
| "num_tokens": 24961698.0, |
| "reward": 0.8686231374740601, |
| "reward_std": 0.19413742423057556, |
| "rewards/cross_solution_unittest_reward/mean": 0.43431156873703003, |
| "rewards/cross_solution_unittest_reward/std": 0.26492878049612045, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 61 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 919.0, |
| "completions/max_terminated_length": 919.0, |
| "completions/mean_length": 520.0104370117188, |
| "completions/mean_terminated_length": 520.0104370117188, |
| "completions/min_length": 265.0, |
| "completions/min_terminated_length": 265.0, |
| "entropy": 0.1033935546875, |
| "epoch": 0.5657256914741945, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0341770313680172, |
| "kl": 0.004704952239990234, |
| "learning_rate": 8.294778073673762e-06, |
| "loss": 0.0204, |
| "num_tokens": 25386422.0, |
| "reward": 0.747821033000946, |
| "reward_std": 0.21126197278499603, |
| "rewards/cross_solution_unittest_reward/mean": 0.37391049042344093, |
| "rewards/cross_solution_unittest_reward/std": 0.19313736259937286, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 62 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 802.0, |
| "completions/max_terminated_length": 802.0, |
| "completions/mean_length": 457.11199951171875, |
| "completions/mean_terminated_length": 457.11199951171875, |
| "completions/min_length": 234.0, |
| "completions/min_terminated_length": 234.0, |
| "entropy": 0.1014862060546875, |
| "epoch": 0.5748502994011976, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03489193320274353, |
| "kl": 0.004260540008544922, |
| "learning_rate": 8.014095333542548e-06, |
| "loss": -0.0047, |
| "num_tokens": 25751057.0, |
| "reward": 0.7548073530197144, |
| "reward_std": 0.20320609211921692, |
| "rewards/cross_solution_unittest_reward/mean": 0.3774036541581154, |
| "rewards/cross_solution_unittest_reward/std": 0.21862108260393143, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 63 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 842.0, |
| "completions/mean_length": 497.3567810058594, |
| "completions/mean_terminated_length": 489.29766845703125, |
| "completions/min_length": 259.0, |
| "completions/min_terminated_length": 259.0, |
| "entropy": 0.09716796875, |
| "epoch": 0.5839749073282008, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04765147343277931, |
| "kl": 0.00534820556640625, |
| "learning_rate": 7.735032325742355e-06, |
| "loss": -0.0042, |
| "num_tokens": 26145418.0, |
| "reward": 1.0832295417785645, |
| "reward_std": 0.20443449914455414, |
| "rewards/cross_solution_unittest_reward/mean": 0.5416147857904434, |
| "rewards/cross_solution_unittest_reward/std": 0.1956881359219551, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 64 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1014.0, |
| "completions/max_terminated_length": 1014.0, |
| "completions/mean_length": 469.390625, |
| "completions/mean_terminated_length": 469.390625, |
| "completions/min_length": 178.0, |
| "completions/min_terminated_length": 178.0, |
| "entropy": 0.110504150390625, |
| "epoch": 0.5930995152552039, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033948320895433426, |
| "kl": 0.004923343658447266, |
| "learning_rate": 7.4578166580651335e-06, |
| "loss": -0.0063, |
| "num_tokens": 26531240.0, |
| "reward": 0.9129461050033569, |
| "reward_std": 0.2018149346113205, |
| "rewards/cross_solution_unittest_reward/mean": 0.4564730077981949, |
| "rewards/cross_solution_unittest_reward/std": 0.2923680394887924, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 65 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.953125, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1764.0, |
| "completions/mean_length": 536.3333740234375, |
| "completions/mean_terminated_length": 512.3359375, |
| "completions/min_length": 253.0, |
| "completions/min_terminated_length": 253.0, |
| "entropy": 0.0997772216796875, |
| "epoch": 0.602224123182207, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030758032575249672, |
| "kl": 0.004119873046875, |
| "learning_rate": 7.182674431585703e-06, |
| "loss": 0.0112, |
| "num_tokens": 27009184.0, |
| "reward": 0.9593736529350281, |
| "reward_std": 0.22100883722305298, |
| "rewards/cross_solution_unittest_reward/mean": 0.47968684136867523, |
| "rewards/cross_solution_unittest_reward/std": 0.2801002562046051, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 66 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 792.0, |
| "completions/max_terminated_length": 792.0, |
| "completions/mean_length": 486.00262451171875, |
| "completions/mean_terminated_length": 486.00262451171875, |
| "completions/min_length": 243.0, |
| "completions/min_terminated_length": 243.0, |
| "entropy": 0.0943145751953125, |
| "epoch": 0.6113487311092102, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03071185201406479, |
| "kl": 0.004786491394042969, |
| "learning_rate": 6.909830056250527e-06, |
| "loss": 0.0004, |
| "num_tokens": 27460937.0, |
| "reward": 1.0903098583221436, |
| "reward_std": 0.25236761569976807, |
| "rewards/cross_solution_unittest_reward/mean": 0.5451549142599106, |
| "rewards/cross_solution_unittest_reward/std": 0.24181817471981049, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 67 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1091.0, |
| "completions/max_terminated_length": 1091.0, |
| "completions/mean_length": 514.515625, |
| "completions/mean_terminated_length": 514.515625, |
| "completions/min_length": 257.0, |
| "completions/min_terminated_length": 257.0, |
| "entropy": 0.0963897705078125, |
| "epoch": 0.6204733390362133, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030294137075543404, |
| "kl": 0.0035681724548339844, |
| "learning_rate": 6.639506067845698e-06, |
| "loss": 0.0073, |
| "num_tokens": 27860791.0, |
| "reward": 0.9012066721916199, |
| "reward_std": 0.18023371696472168, |
| "rewards/cross_solution_unittest_reward/mean": 0.45060331374406815, |
| "rewards/cross_solution_unittest_reward/std": 0.19457216560840607, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 68 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 2792.0, |
| "completions/max_terminated_length": 2792.0, |
| "completions/mean_length": 488.8046875, |
| "completions/mean_terminated_length": 488.8046875, |
| "completions/min_length": 251.0, |
| "completions/min_terminated_length": 251.0, |
| "entropy": 0.100738525390625, |
| "epoch": 0.6295979469632165, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029943598434329033, |
| "kl": 0.004576683044433594, |
| "learning_rate": 6.3719229464935915e-06, |
| "loss": 0.0022, |
| "num_tokens": 28294524.0, |
| "reward": 0.9089535474777222, |
| "reward_std": 0.21831755340099335, |
| "rewards/cross_solution_unittest_reward/mean": 0.4544767662882805, |
| "rewards/cross_solution_unittest_reward/std": 0.20777993649244308, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 69 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1007.0, |
| "completions/max_terminated_length": 1007.0, |
| "completions/mean_length": 517.0234375, |
| "completions/mean_terminated_length": 517.0234375, |
| "completions/min_length": 252.0, |
| "completions/min_terminated_length": 252.0, |
| "entropy": 0.09405517578125, |
| "epoch": 0.6387225548902196, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032788679003715515, |
| "kl": 0.00408935546875, |
| "learning_rate": 6.107298936826086e-06, |
| "loss": 0.003, |
| "num_tokens": 28738997.0, |
| "reward": 0.9689019918441772, |
| "reward_std": 0.2762044668197632, |
| "rewards/cross_solution_unittest_reward/mean": 0.4844510108232498, |
| "rewards/cross_solution_unittest_reward/std": 0.21708913147449493, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 70 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 898.0, |
| "completions/max_terminated_length": 898.0, |
| "completions/mean_length": 492.09637451171875, |
| "completions/mean_terminated_length": 492.09637451171875, |
| "completions/min_length": 280.0, |
| "completions/min_terminated_length": 280.0, |
| "entropy": 0.10223388671875, |
| "epoch": 0.6478471628172227, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03405945748090744, |
| "kl": 0.004323005676269531, |
| "learning_rate": 5.845849869981137e-06, |
| "loss": -0.0087, |
| "num_tokens": 29140394.0, |
| "reward": 0.6452049016952515, |
| "reward_std": 0.18944227695465088, |
| "rewards/cross_solution_unittest_reward/mean": 0.32260240986943245, |
| "rewards/cross_solution_unittest_reward/std": 0.19827812910079956, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 71 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1050.0, |
| "completions/max_terminated_length": 1050.0, |
| "completions/mean_length": 511.7578125, |
| "completions/mean_terminated_length": 511.7578125, |
| "completions/min_length": 275.0, |
| "completions/min_terminated_length": 275.0, |
| "entropy": 0.0999755859375, |
| "epoch": 0.6569717707442259, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04706510528922081, |
| "kl": 0.005205631256103516, |
| "learning_rate": 5.587788987567785e-06, |
| "loss": 0.0065, |
| "num_tokens": 29547821.0, |
| "reward": 0.9646930694580078, |
| "reward_std": 0.1975850760936737, |
| "rewards/cross_solution_unittest_reward/mean": 0.4823464900255203, |
| "rewards/cross_solution_unittest_reward/std": 0.26742689311504364, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 72 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 960.0, |
| "completions/max_terminated_length": 960.0, |
| "completions/mean_length": 469.0859375, |
| "completions/mean_terminated_length": 469.0859375, |
| "completions/min_length": 207.0, |
| "completions/min_terminated_length": 207.0, |
| "entropy": 0.103759765625, |
| "epoch": 0.666096378671229, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.062288798391819, |
| "kl": 0.007381916046142578, |
| "learning_rate": 5.333326767743263e-06, |
| "loss": -0.0033, |
| "num_tokens": 29945526.0, |
| "reward": 1.0015891790390015, |
| "reward_std": 0.24343864619731903, |
| "rewards/cross_solution_unittest_reward/mean": 0.5007946044206619, |
| "rewards/cross_solution_unittest_reward/std": 0.24663101881742477, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 73 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 970.0, |
| "completions/max_terminated_length": 970.0, |
| "completions/mean_length": 514.5989990234375, |
| "completions/mean_terminated_length": 514.5989990234375, |
| "completions/min_length": 197.0, |
| "completions/min_terminated_length": 197.0, |
| "entropy": 0.0988922119140625, |
| "epoch": 0.6752209865982322, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.034228503704071045, |
| "kl": 0.004460811614990234, |
| "learning_rate": 5.082670753543961e-06, |
| "loss": 0.0019, |
| "num_tokens": 30349572.0, |
| "reward": 0.8880095481872559, |
| "reward_std": 0.2291492521762848, |
| "rewards/cross_solution_unittest_reward/mean": 0.44400477409362793, |
| "rewards/cross_solution_unittest_reward/std": 0.24787399172782898, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 74 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1092.0, |
| "completions/mean_length": 510.9739685058594, |
| "completions/mean_terminated_length": 502.9504089355469, |
| "completions/min_length": 277.0, |
| "completions/min_terminated_length": 277.0, |
| "entropy": 0.106231689453125, |
| "epoch": 0.6843455945252352, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030502676963806152, |
| "kl": 0.003562450408935547, |
| "learning_rate": 4.836025383610382e-06, |
| "loss": -0.0003, |
| "num_tokens": 30727274.0, |
| "reward": 0.9304172396659851, |
| "reward_std": 0.20447766780853271, |
| "rewards/cross_solution_unittest_reward/mean": 0.46520860493183136, |
| "rewards/cross_solution_unittest_reward/std": 0.23417390882968903, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 75 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 888.0, |
| "completions/max_terminated_length": 888.0, |
| "completions/mean_length": 464.9453125, |
| "completions/mean_terminated_length": 464.9453125, |
| "completions/min_length": 217.0, |
| "completions/min_terminated_length": 217.0, |
| "entropy": 0.1075897216796875, |
| "epoch": 0.6934702024522383, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03586795926094055, |
| "kl": 0.004437446594238281, |
| "learning_rate": 4.593591825444028e-06, |
| "loss": 0.0051, |
| "num_tokens": 31071701.0, |
| "reward": 0.7625653147697449, |
| "reward_std": 0.1792157143354416, |
| "rewards/cross_solution_unittest_reward/mean": 0.38128262013196945, |
| "rewards/cross_solution_unittest_reward/std": 0.22236895561218262, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 76 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 890.0, |
| "completions/mean_length": 497.3125, |
| "completions/mean_terminated_length": 481.1518249511719, |
| "completions/min_length": 241.0, |
| "completions/min_terminated_length": 241.0, |
| "entropy": 0.105224609375, |
| "epoch": 0.7025948103792415, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03737795352935791, |
| "kl": 0.003971576690673828, |
| "learning_rate": 4.355567811332311e-06, |
| "loss": -0.0003, |
| "num_tokens": 31477149.0, |
| "reward": 0.9584690928459167, |
| "reward_std": 0.2223634421825409, |
| "rewards/cross_solution_unittest_reward/mean": 0.47923456132411957, |
| "rewards/cross_solution_unittest_reward/std": 0.20396708697080612, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 77 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 880.0, |
| "completions/max_terminated_length": 880.0, |
| "completions/mean_length": 495.7161560058594, |
| "completions/mean_terminated_length": 495.7161560058594, |
| "completions/min_length": 236.0, |
| "completions/min_terminated_length": 236.0, |
| "entropy": 0.097930908203125, |
| "epoch": 0.7117194183062446, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.159642294049263, |
| "kl": 0.006262063980102539, |
| "learning_rate": 4.12214747707527e-06, |
| "loss": -0.0001, |
| "num_tokens": 31912712.0, |
| "reward": 0.9255527257919312, |
| "reward_std": 0.2571796178817749, |
| "rewards/cross_solution_unittest_reward/mean": 0.46277637779712677, |
| "rewards/cross_solution_unittest_reward/std": 0.25626206398010254, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 78 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.9375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1027.0, |
| "completions/mean_length": 512.7135620117188, |
| "completions/mean_terminated_length": 480.38421630859375, |
| "completions/min_length": 224.0, |
| "completions/min_terminated_length": 224.0, |
| "entropy": 0.1019744873046875, |
| "epoch": 0.7208440262332478, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032687410712242126, |
| "kl": 0.004110813140869141, |
| "learning_rate": 3.893521203645618e-06, |
| "loss": -0.0029, |
| "num_tokens": 32308370.0, |
| "reward": 0.9185789823532104, |
| "reward_std": 0.19305042922496796, |
| "rewards/cross_solution_unittest_reward/mean": 0.45928946137428284, |
| "rewards/cross_solution_unittest_reward/std": 0.23322047293186188, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 79 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 938.0, |
| "completions/max_terminated_length": 938.0, |
| "completions/mean_length": 471.16668701171875, |
| "completions/mean_terminated_length": 471.16668701171875, |
| "completions/min_length": 246.0, |
| "completions/min_terminated_length": 246.0, |
| "entropy": 0.0972900390625, |
| "epoch": 0.7299686341602509, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.034349896013736725, |
| "kl": 0.0041866302490234375, |
| "learning_rate": 3.6698754619112974e-06, |
| "loss": -0.0032, |
| "num_tokens": 32667882.0, |
| "reward": 0.8720422983169556, |
| "reward_std": 0.18141961097717285, |
| "rewards/cross_solution_unittest_reward/mean": 0.436021126806736, |
| "rewards/cross_solution_unittest_reward/std": 0.23245983570814133, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 80 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 952.0, |
| "completions/max_terminated_length": 952.0, |
| "completions/mean_length": 484.87762451171875, |
| "completions/mean_terminated_length": 484.87762451171875, |
| "completions/min_length": 194.0, |
| "completions/min_terminated_length": 194.0, |
| "entropy": 0.099090576171875, |
| "epoch": 0.739093242087254, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03139191120862961, |
| "kl": 0.0039272308349609375, |
| "learning_rate": 3.4513926605471504e-06, |
| "loss": 0.0016, |
| "num_tokens": 33050987.0, |
| "reward": 0.9457516074180603, |
| "reward_std": 0.1805034577846527, |
| "rewards/cross_solution_unittest_reward/mean": 0.47287583351135254, |
| "rewards/cross_solution_unittest_reward/std": 0.19942139089107513, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 81 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1118.0, |
| "completions/max_terminated_length": 1118.0, |
| "completions/mean_length": 475.60418701171875, |
| "completions/mean_terminated_length": 475.60418701171875, |
| "completions/min_length": 212.0, |
| "completions/min_terminated_length": 212.0, |
| "entropy": 0.1024169921875, |
| "epoch": 0.7482178500142572, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030605316162109375, |
| "kl": 0.004054546356201172, |
| "learning_rate": 3.2382509972598087e-06, |
| "loss": 0.0052, |
| "num_tokens": 33439955.0, |
| "reward": 0.8594505190849304, |
| "reward_std": 0.20357158780097961, |
| "rewards/cross_solution_unittest_reward/mean": 0.4297252967953682, |
| "rewards/cross_solution_unittest_reward/std": 0.24937064200639725, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 82 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 808.0, |
| "completions/mean_length": 483.18231201171875, |
| "completions/mean_terminated_length": 475.086181640625, |
| "completions/min_length": 264.0, |
| "completions/min_terminated_length": 264.0, |
| "entropy": 0.101776123046875, |
| "epoch": 0.7573424579412603, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.034236159175634384, |
| "kl": 0.004290103912353516, |
| "learning_rate": 3.0306243134470668e-06, |
| "loss": 0.0102, |
| "num_tokens": 33834753.0, |
| "reward": 0.8233698606491089, |
| "reward_std": 0.25579404830932617, |
| "rewards/cross_solution_unittest_reward/mean": 0.41168488562107086, |
| "rewards/cross_solution_unittest_reward/std": 0.23754408955574036, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 83 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1117.0, |
| "completions/mean_length": 483.5625, |
| "completions/mean_terminated_length": 467.329833984375, |
| "completions/min_length": 234.0, |
| "completions/min_terminated_length": 234.0, |
| "entropy": 0.110076904296875, |
| "epoch": 0.7664670658682635, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03293149545788765, |
| "kl": 0.0037369728088378906, |
| "learning_rate": 2.8286819524103657e-06, |
| "loss": -0.0012, |
| "num_tokens": 34176409.0, |
| "reward": 0.8954019546508789, |
| "reward_std": 0.20441854000091553, |
| "rewards/cross_solution_unittest_reward/mean": 0.44770099222660065, |
| "rewards/cross_solution_unittest_reward/std": 0.21769069880247116, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 84 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 937.0, |
| "completions/max_terminated_length": 937.0, |
| "completions/mean_length": 485.5442810058594, |
| "completions/mean_terminated_length": 485.5442810058594, |
| "completions/min_length": 242.0, |
| "completions/min_terminated_length": 242.0, |
| "entropy": 0.0960693359375, |
| "epoch": 0.7755916737952666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033041298389434814, |
| "kl": 0.004325389862060547, |
| "learning_rate": 2.6325886212359496e-06, |
| "loss": 0.0025, |
| "num_tokens": 34602482.0, |
| "reward": 0.9020225405693054, |
| "reward_std": 0.16266906261444092, |
| "rewards/cross_solution_unittest_reward/mean": 0.4510112926363945, |
| "rewards/cross_solution_unittest_reward/std": 0.19898167252540588, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 85 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 826.0, |
| "completions/mean_length": 472.5625, |
| "completions/mean_terminated_length": 464.43865966796875, |
| "completions/min_length": 254.0, |
| "completions/min_terminated_length": 254.0, |
| "entropy": 0.106231689453125, |
| "epoch": 0.7847162817222697, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03541736304759979, |
| "kl": 0.005221366882324219, |
| "learning_rate": 2.4425042564574186e-06, |
| "loss": -0.0027, |
| "num_tokens": 35012690.0, |
| "reward": 0.8730189204216003, |
| "reward_std": 0.19061098992824554, |
| "rewards/cross_solution_unittest_reward/mean": 0.4365094378590584, |
| "rewards/cross_solution_unittest_reward/std": 0.19948572665452957, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 86 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.953125, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 932.0, |
| "completions/mean_length": 523.3880615234375, |
| "completions/mean_terminated_length": 499.2887268066406, |
| "completions/min_length": 246.0, |
| "completions/min_terminated_length": 246.0, |
| "entropy": 0.1020660400390625, |
| "epoch": 0.7938408896492729, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.035699885338544846, |
| "kl": 0.0038251876831054688, |
| "learning_rate": 2.2585838936091753e-06, |
| "loss": 0.0055, |
| "num_tokens": 35446079.0, |
| "reward": 0.9853703379631042, |
| "reward_std": 0.1933068484067917, |
| "rewards/cross_solution_unittest_reward/mean": 0.49268509447574615, |
| "rewards/cross_solution_unittest_reward/std": 0.2293217033147812, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 87 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 949.0, |
| "completions/max_terminated_length": 949.0, |
| "completions/mean_length": 436.2578125, |
| "completions/mean_terminated_length": 436.2578125, |
| "completions/min_length": 236.0, |
| "completions/min_terminated_length": 236.0, |
| "entropy": 0.1071624755859375, |
| "epoch": 0.802965497576276, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.12767620384693146, |
| "kl": 0.010437488555908203, |
| "learning_rate": 2.0809775407772505e-06, |
| "loss": 0.0142, |
| "num_tokens": 35801002.0, |
| "reward": 0.9056326746940613, |
| "reward_std": 0.22697775065898895, |
| "rewards/cross_solution_unittest_reward/mean": 0.45281632244586945, |
| "rewards/cross_solution_unittest_reward/std": 0.2281804159283638, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 88 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 821.0, |
| "completions/max_terminated_length": 821.0, |
| "completions/mean_length": 501.71356201171875, |
| "completions/mean_terminated_length": 501.71356201171875, |
| "completions/min_length": 316.0, |
| "completions/min_terminated_length": 316.0, |
| "entropy": 0.0912628173828125, |
| "epoch": 0.8120901055032792, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030100375413894653, |
| "kl": 0.004030466079711914, |
| "learning_rate": 1.9098300562505266e-06, |
| "loss": 0.0095, |
| "num_tokens": 36223060.0, |
| "reward": 0.8723653554916382, |
| "reward_std": 0.1536586731672287, |
| "rewards/cross_solution_unittest_reward/mean": 0.4361826628446579, |
| "rewards/cross_solution_unittest_reward/std": 0.24371575564146042, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 89 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 846.0, |
| "completions/max_terminated_length": 846.0, |
| "completions/mean_length": 482.5078125, |
| "completions/mean_terminated_length": 482.5078125, |
| "completions/min_length": 239.0, |
| "completions/min_terminated_length": 239.0, |
| "entropy": 0.096923828125, |
| "epoch": 0.8212147134302823, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.040627796202898026, |
| "kl": 0.007803916931152344, |
| "learning_rate": 1.74528103037226e-06, |
| "loss": -0.0038, |
| "num_tokens": 36635343.0, |
| "reward": 0.7952584028244019, |
| "reward_std": 0.23465335369110107, |
| "rewards/cross_solution_unittest_reward/mean": 0.39762915670871735, |
| "rewards/cross_solution_unittest_reward/std": 0.2743668779730797, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 90 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1028.0, |
| "completions/max_terminated_length": 1028.0, |
| "completions/mean_length": 482.26824951171875, |
| "completions/mean_terminated_length": 482.26824951171875, |
| "completions/min_length": 228.0, |
| "completions/min_terminated_length": 228.0, |
| "entropy": 0.1040496826171875, |
| "epoch": 0.8303393213572854, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.036647334694862366, |
| "kl": 0.0043468475341796875, |
| "learning_rate": 1.587464671688187e-06, |
| "loss": -0.005, |
| "num_tokens": 37027198.0, |
| "reward": 0.8686568140983582, |
| "reward_std": 0.1460837721824646, |
| "rewards/cross_solution_unittest_reward/mean": 0.4343283474445343, |
| "rewards/cross_solution_unittest_reward/std": 0.21033113449811935, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 91 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 941.0, |
| "completions/mean_length": 507.2317810058594, |
| "completions/mean_terminated_length": 491.123046875, |
| "completions/min_length": 206.0, |
| "completions/min_terminated_length": 206.0, |
| "entropy": 0.09820556640625, |
| "epoch": 0.8394639292842886, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033177897334098816, |
| "kl": 0.004420280456542969, |
| "learning_rate": 1.436509697484111e-06, |
| "loss": -0.001, |
| "num_tokens": 37446527.0, |
| "reward": 0.8312811851501465, |
| "reward_std": 0.22770294547080994, |
| "rewards/cross_solution_unittest_reward/mean": 0.41564057022333145, |
| "rewards/cross_solution_unittest_reward/std": 0.25115445256233215, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 92 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.921875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1317.0, |
| "completions/mean_length": 578.09375, |
| "completions/mean_terminated_length": 538.43798828125, |
| "completions/min_length": 288.0, |
| "completions/min_terminated_length": 288.0, |
| "entropy": 0.086181640625, |
| "epoch": 0.8485885372112917, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0323200523853302, |
| "kl": 0.004235267639160156, |
| "learning_rate": 1.2925392288022299e-06, |
| "loss": 0.0244, |
| "num_tokens": 37887771.0, |
| "reward": 0.7843211889266968, |
| "reward_std": 0.22614744305610657, |
| "rewards/cross_solution_unittest_reward/mean": 0.3921606168150902, |
| "rewards/cross_solution_unittest_reward/std": 0.23859115689992905, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 93 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 961.0, |
| "completions/mean_length": 527.734375, |
| "completions/mean_terminated_length": 519.7545776367188, |
| "completions/min_length": 171.0, |
| "completions/min_terminated_length": 171.0, |
| "entropy": 0.089080810546875, |
| "epoch": 0.8577131451382949, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03161383792757988, |
| "kl": 0.0037050247192382812, |
| "learning_rate": 1.1556706900218572e-06, |
| "loss": -0.0029, |
| "num_tokens": 38306421.0, |
| "reward": 0.8312766551971436, |
| "reward_std": 0.25127798318862915, |
| "rewards/cross_solution_unittest_reward/mean": 0.4156383126974106, |
| "rewards/cross_solution_unittest_reward/std": 0.27913951128721237, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 94 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 762.0, |
| "completions/max_terminated_length": 762.0, |
| "completions/mean_length": 482.65106201171875, |
| "completions/mean_terminated_length": 482.65106201171875, |
| "completions/min_length": 243.0, |
| "completions/min_terminated_length": 243.0, |
| "entropy": 0.096649169921875, |
| "epoch": 0.866837753065298, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030079808086156845, |
| "kl": 0.004782915115356445, |
| "learning_rate": 1.0260157130864178e-06, |
| "loss": 0.0004, |
| "num_tokens": 38748439.0, |
| "reward": 0.8986494541168213, |
| "reward_std": 0.20043903589248657, |
| "rewards/cross_solution_unittest_reward/mean": 0.44932468235492706, |
| "rewards/cross_solution_unittest_reward/std": 0.1927783116698265, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 95 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 878.0, |
| "completions/max_terminated_length": 878.0, |
| "completions/mean_length": 512.8932495117188, |
| "completions/mean_terminated_length": 512.8932495117188, |
| "completions/min_length": 225.0, |
| "completions/min_terminated_length": 225.0, |
| "entropy": 0.0985107421875, |
| "epoch": 0.8759623609923011, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.035013340413570404, |
| "kl": 0.004712104797363281, |
| "learning_rate": 9.036800464548157e-07, |
| "loss": 0.0024, |
| "num_tokens": 39133230.0, |
| "reward": 0.8117542266845703, |
| "reward_std": 0.20024093985557556, |
| "rewards/cross_solution_unittest_reward/mean": 0.40587710589170456, |
| "rewards/cross_solution_unittest_reward/std": 0.2243197336792946, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 96 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 811.0, |
| "completions/mean_length": 455.43231201171875, |
| "completions/mean_terminated_length": 447.2637023925781, |
| "completions/min_length": 228.0, |
| "completions/min_terminated_length": 228.0, |
| "entropy": 0.1064453125, |
| "epoch": 0.8850869689193043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03698151558637619, |
| "kl": 0.004943370819091797, |
| "learning_rate": 7.887634688515e-07, |
| "loss": 0.0116, |
| "num_tokens": 39453516.0, |
| "reward": 1.0091822147369385, |
| "reward_std": 0.17214202880859375, |
| "rewards/cross_solution_unittest_reward/mean": 0.5045911073684692, |
| "rewards/cross_solution_unittest_reward/std": 0.2308211475610733, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 97 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 905.0, |
| "completions/max_terminated_length": 905.0, |
| "completions/mean_length": 517.7213745117188, |
| "completions/mean_terminated_length": 517.7213745117188, |
| "completions/min_length": 332.0, |
| "completions/min_terminated_length": 332.0, |
| "entropy": 0.09765625, |
| "epoch": 0.8942115768463074, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0317588746547699, |
| "kl": 0.004177570343017578, |
| "learning_rate": 6.813597078854772e-07, |
| "loss": -0.0053, |
| "num_tokens": 39895593.0, |
| "reward": 0.9603371620178223, |
| "reward_std": 0.18226468563079834, |
| "rewards/cross_solution_unittest_reward/mean": 0.48016858100891113, |
| "rewards/cross_solution_unittest_reward/std": 0.2831304222345352, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 98 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 795.0, |
| "completions/max_terminated_length": 795.0, |
| "completions/mean_length": 476.8515625, |
| "completions/mean_terminated_length": 476.8515625, |
| "completions/min_length": 175.0, |
| "completions/min_terminated_length": 175.0, |
| "entropy": 0.1009063720703125, |
| "epoch": 0.9033361847733106, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03179348632693291, |
| "kl": 0.004183292388916016, |
| "learning_rate": 5.815563636047539e-07, |
| "loss": -0.0026, |
| "num_tokens": 40260864.0, |
| "reward": 0.9521991014480591, |
| "reward_std": 0.2101556956768036, |
| "rewards/cross_solution_unittest_reward/mean": 0.47609956562519073, |
| "rewards/cross_solution_unittest_reward/std": 0.225713312625885, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 99 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 798.0, |
| "completions/max_terminated_length": 798.0, |
| "completions/mean_length": 457.1015625, |
| "completions/mean_terminated_length": 457.1015625, |
| "completions/min_length": 207.0, |
| "completions/min_terminated_length": 207.0, |
| "entropy": 0.1097564697265625, |
| "epoch": 0.9124607927003137, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03737381473183632, |
| "kl": 0.0048885345458984375, |
| "learning_rate": 4.894348370484648e-07, |
| "loss": 0.0142, |
| "num_tokens": 40678271.0, |
| "reward": 0.8324875235557556, |
| "reward_std": 0.20148925483226776, |
| "rewards/cross_solution_unittest_reward/mean": 0.416243739426136, |
| "rewards/cross_solution_unittest_reward/std": 0.25171586126089096, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 100 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1004.0, |
| "completions/mean_length": 483.8359375, |
| "completions/mean_terminated_length": 475.74151611328125, |
| "completions/min_length": 254.0, |
| "completions/min_terminated_length": 254.0, |
| "entropy": 0.09332275390625, |
| "epoch": 0.9215854006273168, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030489500612020493, |
| "kl": 0.004380226135253906, |
| "learning_rate": 4.0507026385502747e-07, |
| "loss": 0.0107, |
| "num_tokens": 41066632.0, |
| "reward": 0.9275814890861511, |
| "reward_std": 0.1952568143606186, |
| "rewards/cross_solution_unittest_reward/mean": 0.46379074454307556, |
| "rewards/cross_solution_unittest_reward/std": 0.20886488258838654, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 101 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 876.0, |
| "completions/max_terminated_length": 876.0, |
| "completions/mean_length": 490.8802185058594, |
| "completions/mean_terminated_length": 490.8802185058594, |
| "completions/min_length": 230.0, |
| "completions/min_terminated_length": 230.0, |
| "entropy": 0.0925445556640625, |
| "epoch": 0.93071000855432, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02890767715871334, |
| "kl": 0.004089832305908203, |
| "learning_rate": 3.2853145298042954e-07, |
| "loss": 0.0088, |
| "num_tokens": 41490450.0, |
| "reward": 1.0289883613586426, |
| "reward_std": 0.23402146995067596, |
| "rewards/cross_solution_unittest_reward/mean": 0.5144941955804825, |
| "rewards/cross_solution_unittest_reward/std": 0.20954615622758865, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 102 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1279.0, |
| "completions/max_terminated_length": 1279.0, |
| "completions/mean_length": 471.7421875, |
| "completions/mean_terminated_length": 471.7421875, |
| "completions/min_length": 234.0, |
| "completions/min_terminated_length": 234.0, |
| "entropy": 0.100799560546875, |
| "epoch": 0.939834616481323, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.039594829082489014, |
| "kl": 0.005157470703125, |
| "learning_rate": 2.5988083057666534e-07, |
| "loss": 0.0164, |
| "num_tokens": 41862519.0, |
| "reward": 1.0344469547271729, |
| "reward_std": 0.2174767255783081, |
| "rewards/cross_solution_unittest_reward/mean": 0.5172234624624252, |
| "rewards/cross_solution_unittest_reward/std": 0.2513153925538063, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 103 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 833.0, |
| "completions/max_terminated_length": 833.0, |
| "completions/mean_length": 489.171875, |
| "completions/mean_terminated_length": 489.171875, |
| "completions/min_length": 246.0, |
| "completions/min_terminated_length": 246.0, |
| "entropy": 0.0948944091796875, |
| "epoch": 0.9489592244083263, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04726433753967285, |
| "kl": 0.005943775177001953, |
| "learning_rate": 1.9917438907606556e-07, |
| "loss": 0.0073, |
| "num_tokens": 42276585.0, |
| "reward": 0.8814212679862976, |
| "reward_std": 0.1902289092540741, |
| "rewards/cross_solution_unittest_reward/mean": 0.4407106339931488, |
| "rewards/cross_solution_unittest_reward/std": 0.1961411088705063, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 104 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 737.0, |
| "completions/max_terminated_length": 737.0, |
| "completions/mean_length": 453.78125, |
| "completions/mean_terminated_length": 453.78125, |
| "completions/min_length": 216.0, |
| "completions/min_terminated_length": 216.0, |
| "entropy": 0.10247802734375, |
| "epoch": 0.9580838323353293, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0373138003051281, |
| "kl": 0.004960536956787109, |
| "learning_rate": 1.464616415230702e-07, |
| "loss": -0.0085, |
| "num_tokens": 42634949.0, |
| "reward": 0.7063878774642944, |
| "reward_std": 0.22609077394008636, |
| "rewards/cross_solution_unittest_reward/mean": 0.3531939312815666, |
| "rewards/cross_solution_unittest_reward/std": 0.23600679636001587, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 105 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 814.0, |
| "completions/max_terminated_length": 814.0, |
| "completions/mean_length": 514.3932495117188, |
| "completions/mean_terminated_length": 514.3932495117188, |
| "completions/min_length": 289.0, |
| "completions/min_terminated_length": 289.0, |
| "entropy": 0.0894622802734375, |
| "epoch": 0.9672084402623324, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028397122398018837, |
| "kl": 0.004255771636962891, |
| "learning_rate": 1.0178558119067316e-07, |
| "loss": -0.008, |
| "num_tokens": 43057844.0, |
| "reward": 0.8395287990570068, |
| "reward_std": 0.23434078693389893, |
| "rewards/cross_solution_unittest_reward/mean": 0.4197644293308258, |
| "rewards/cross_solution_unittest_reward/std": 0.25368085503578186, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 106 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 2142.0, |
| "completions/max_terminated_length": 2142.0, |
| "completions/mean_length": 488.59375, |
| "completions/mean_terminated_length": 488.59375, |
| "completions/min_length": 283.0, |
| "completions/min_terminated_length": 283.0, |
| "entropy": 0.10888671875, |
| "epoch": 0.9763330481893356, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.034389153122901917, |
| "kl": 0.004794120788574219, |
| "learning_rate": 6.51826465144978e-08, |
| "loss": 0.003, |
| "num_tokens": 43460008.0, |
| "reward": 0.9071213603019714, |
| "reward_std": 0.1905367076396942, |
| "rewards/cross_solution_unittest_reward/mean": 0.4535607025027275, |
| "rewards/cross_solution_unittest_reward/std": 0.2374211624264717, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 107 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1084.0, |
| "completions/max_terminated_length": 1084.0, |
| "completions/mean_length": 535.3229370117188, |
| "completions/mean_terminated_length": 535.3229370117188, |
| "completions/min_length": 228.0, |
| "completions/min_terminated_length": 228.0, |
| "entropy": 0.0862884521484375, |
| "epoch": 0.9854576561163387, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029783420264720917, |
| "kl": 0.0033812522888183594, |
| "learning_rate": 3.668269137308666e-08, |
| "loss": -0.0022, |
| "num_tokens": 43903588.0, |
| "reward": 1.0605829954147339, |
| "reward_std": 0.17206411063671112, |
| "rewards/cross_solution_unittest_reward/mean": 0.5302915126085281, |
| "rewards/cross_solution_unittest_reward/std": 0.2089696228504181, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 108 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 940.0, |
| "completions/max_terminated_length": 940.0, |
| "completions/mean_length": 456.51824951171875, |
| "completions/mean_terminated_length": 456.51824951171875, |
| "completions/min_length": 188.0, |
| "completions/min_terminated_length": 188.0, |
| "entropy": 0.1035308837890625, |
| "epoch": 0.9945822640433419, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.034084659069776535, |
| "kl": 0.004221439361572266, |
| "learning_rate": 1.630896073864352e-08, |
| "loss": 0.0052, |
| "num_tokens": 44271795.0, |
| "reward": 1.0674467086791992, |
| "reward_std": 0.20672927796840668, |
| "rewards/cross_solution_unittest_reward/mean": 0.5337233692407608, |
| "rewards/cross_solution_unittest_reward/std": 0.2225872203707695, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 109 |
| }, |
| { |
| "epoch": 0.9945822640433419, |
| "step": 109, |
| "total_flos": 0.0, |
| "train_loss": 0.0018782940647274324, |
| "train_runtime": 42443.7986, |
| "train_samples_per_second": 0.031, |
| "train_steps_per_second": 0.003 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 110, |
| "num_input_tokens_seen": 44271795, |
| "num_train_epochs": 1, |
| "save_steps": 25, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
