diff --git "a/trainer_state.json" "b/trainer_state.json" --- "a/trainer_state.json" +++ "b/trainer_state.json" @@ -2,9 +2,9 @@ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, - "epoch": 1.8303393213572854, + "epoch": 0.9945822640433419, "eval_steps": 500, - "global_step": 200, + "global_step": 109, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, @@ -16,24 +16,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1070.0, - "completions/max_terminated_length": 1070.0, - "completions/mean_length": 533.8359375, - "completions/mean_terminated_length": 533.8359375, - "completions/min_length": 242.0, - "completions/min_terminated_length": 242.0, - "entropy": 0.0727691650390625, + "completions/max_length": 982.0, + "completions/max_terminated_length": 982.0, + "completions/mean_length": 520.5390625, + "completions/mean_terminated_length": 520.5390625, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "entropy": 0.114654541015625, "epoch": 0.009124607927003137, "frac_reward_zero_std": 0.0, - "grad_norm": 0.023102419450879097, - "kl": 0.00021660327911376953, - "learning_rate": 0.0, - "loss": -0.0135, - "num_tokens": 413577.0, - "reward": 0.6045712828636169, - "reward_std": 0.06758441776037216, - "rewards/cross_solution_unittest/mean": 0.6045712828636169, - "rewards/cross_solution_unittest/std": 0.09926356375217438, + "grad_norm": 0.0243235994130373, + "kl": 0.00023239850997924805, + "learning_rate": 2e-05, + "loss": -0.0022, + "num_tokens": 408495.0, + "reward": 0.963108241558075, + "reward_std": 0.2168295979499817, + "rewards/cross_solution_unittest_reward/mean": 0.48155416548252106, + "rewards/cross_solution_unittest_reward/std": 0.22725073248147964, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 1 @@ -45,24 +45,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1010.0, - "completions/max_terminated_length": 1010.0, - "completions/mean_length": 495.5234375, - "completions/mean_terminated_length": 495.5234375, - "completions/min_length": 219.0, - "completions/min_terminated_length": 219.0, - "entropy": 0.07559967041015625, + "completions/max_length": 1246.0, + "completions/max_terminated_length": 1246.0, + "completions/mean_length": 493.14324951171875, + "completions/mean_terminated_length": 493.14324951171875, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "entropy": 0.1129150390625, "epoch": 0.018249215854006275, "frac_reward_zero_std": 0.0, - "grad_norm": 0.028239840641617775, - "kl": 0.00024643540382385254, - "learning_rate": 1.8181818181818183e-06, - "loss": -0.0058, - "num_tokens": 824930.0, - "reward": 0.5676477551460266, - "reward_std": 0.07896548509597778, - "rewards/cross_solution_unittest/mean": 0.5676476955413818, - "rewards/cross_solution_unittest/std": 0.12314769625663757, + "grad_norm": 0.02810436487197876, + "kl": 0.00023671984672546387, + "learning_rate": 1.9995921928281893e-05, + "loss": -0.004, + "num_tokens": 819398.0, + "reward": 0.7592691779136658, + "reward_std": 0.20917575061321259, + "rewards/cross_solution_unittest_reward/mean": 0.3796345815062523, + "rewards/cross_solution_unittest_reward/std": 0.2533634677529335, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 2 @@ -74,24 +74,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1136.0, - "completions/max_terminated_length": 1136.0, - "completions/mean_length": 568.5729370117188, - "completions/mean_terminated_length": 568.5729370117188, - "completions/min_length": 316.0, - "completions/min_terminated_length": 316.0, - "entropy": 0.0638427734375, + "completions/max_length": 1210.0, + "completions/max_terminated_length": 1210.0, + "completions/mean_length": 584.3802490234375, + "completions/mean_terminated_length": 584.3802490234375, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.09423828125, "epoch": 0.02737382378100941, "frac_reward_zero_std": 0.0, - "grad_norm": 0.026136187836527824, - "kl": 0.00020660459995269775, - "learning_rate": 3.6363636363636366e-06, - "loss": 0.0073, - "num_tokens": 1260498.0, - "reward": 0.5541688203811646, - "reward_std": 0.05321730300784111, - "rewards/cross_solution_unittest/mean": 0.5541687607765198, - "rewards/cross_solution_unittest/std": 0.11517918854951859, + "grad_norm": 0.024342378601431847, + "kl": 0.00020760297775268555, + "learning_rate": 1.9983691039261358e-05, + "loss": -0.0025, + "num_tokens": 1264568.0, + "reward": 0.8243939280509949, + "reward_std": 0.19790497422218323, + "rewards/cross_solution_unittest_reward/mean": 0.41219695657491684, + "rewards/cross_solution_unittest_reward/std": 0.20426170527935028, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 3 @@ -102,25 +102,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, + "completions/clipped_ratio": -4.96875, "completions/max_length": 3584.0, - "completions/max_terminated_length": 845.0, - "completions/mean_length": 497.7317810058594, - "completions/mean_terminated_length": 489.67364501953125, - "completions/min_length": 297.0, - "completions/min_terminated_length": 297.0, - "entropy": 0.0802001953125, + "completions/max_terminated_length": 902.0, + "completions/mean_length": 510.72137451171875, + "completions/mean_terminated_length": 494.6308898925781, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.12127685546875, "epoch": 0.03649843170801255, "frac_reward_zero_std": 0.0, - "grad_norm": 0.027541937306523323, - "kl": 0.0002409219741821289, - "learning_rate": 5.4545454545454545e-06, - "loss": -0.0068, - "num_tokens": 1696939.0, - "reward": 0.5838931798934937, - "reward_std": 0.07090283930301666, - "rewards/cross_solution_unittest/mean": 0.5838931202888489, - "rewards/cross_solution_unittest/std": 0.10168680548667908, + "grad_norm": 0.02658909745514393, + "kl": 0.00026023387908935547, + "learning_rate": 1.9963317308626916e-05, + "loss": -0.0008, + "num_tokens": 1707901.0, + "reward": 0.6146944165229797, + "reward_std": 0.20338332653045654, + "rewards/cross_solution_unittest_reward/mean": 0.30734721571207047, + "rewards/cross_solution_unittest_reward/std": 0.1820686012506485, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 4 @@ -133,23 +133,23 @@ "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 864.0, - "completions/mean_length": 490.75, - "completions/mean_terminated_length": 482.67364501953125, - "completions/min_length": 263.0, - "completions/min_terminated_length": 263.0, - "entropy": 0.082305908203125, + "completions/max_terminated_length": 1694.0, + "completions/mean_length": 495.9921875, + "completions/mean_terminated_length": 487.92950439453125, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.1229400634765625, "epoch": 0.04562303963501568, "frac_reward_zero_std": 0.0, - "grad_norm": 0.027760116383433342, - "kl": 0.0002455413341522217, - "learning_rate": 7.272727272727273e-06, - "loss": -0.0016, - "num_tokens": 2080287.0, - "reward": 0.6031867861747742, - "reward_std": 0.05878433585166931, - "rewards/cross_solution_unittest/mean": 0.6031867861747742, - "rewards/cross_solution_unittest/std": 0.12897227704524994, + "grad_norm": 0.03194279596209526, + "kl": 0.0002792775630950928, + "learning_rate": 1.99348173534855e-05, + "loss": 0.0193, + "num_tokens": 2092986.0, + "reward": 0.8980245590209961, + "reward_std": 0.2468685805797577, + "rewards/cross_solution_unittest_reward/mean": 0.44901224970817566, + "rewards/cross_solution_unittest_reward/std": 0.2793443500995636, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 5 @@ -160,25 +160,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, + "completions/clipped_ratio": -4.96875, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1842.0, - "completions/mean_length": 544.1614990234375, - "completions/mean_terminated_length": 536.2245483398438, - "completions/min_length": 290.0, - "completions/min_terminated_length": 290.0, - "entropy": 0.077362060546875, + "completions/max_terminated_length": 1340.0, + "completions/mean_length": 548.34375, + "completions/mean_terminated_length": 532.4502563476562, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.1123809814453125, "epoch": 0.05474764756201882, "frac_reward_zero_std": 0.0, - "grad_norm": 0.024699553847312927, - "kl": 0.00022616982460021973, - "learning_rate": 9.090909090909091e-06, - "loss": -0.0098, - "num_tokens": 2505261.0, - "reward": 0.6043218374252319, - "reward_std": 0.049580853432416916, - "rewards/cross_solution_unittest/mean": 0.6043217778205872, - "rewards/cross_solution_unittest/std": 0.08841408789157867, + "grad_norm": 0.024234920740127563, + "kl": 0.0002339184284210205, + "learning_rate": 1.989821441880933e-05, + "loss": 0.0085, + "num_tokens": 2517798.0, + "reward": 0.8548812866210938, + "reward_std": 0.16416706144809723, + "rewards/cross_solution_unittest_reward/mean": 0.42744066566228867, + "rewards/cross_solution_unittest_reward/std": 0.24477335810661316, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 6 @@ -190,24 +190,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1419.0, - "completions/max_terminated_length": 1419.0, - "completions/mean_length": 537.2604370117188, - "completions/mean_terminated_length": 537.2604370117188, - "completions/min_length": 257.0, - "completions/min_terminated_length": 257.0, - "entropy": 0.0787200927734375, + "completions/max_length": 3107.0, + "completions/max_terminated_length": 3107.0, + "completions/mean_length": 531.8958740234375, + "completions/mean_terminated_length": 531.8958740234375, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.1142120361328125, "epoch": 0.06387225548902195, "frac_reward_zero_std": 0.0, - "grad_norm": 0.02539636939764023, - "kl": 0.00023606419563293457, - "learning_rate": 1.0909090909090909e-05, - "loss": -0.01, - "num_tokens": 2952069.0, - "reward": 0.597324788570404, - "reward_std": 0.07878245413303375, - "rewards/cross_solution_unittest/mean": 0.597324788570404, - "rewards/cross_solution_unittest/std": 0.11920402199029922, + "grad_norm": 0.025173544883728027, + "kl": 0.00025781989097595215, + "learning_rate": 1.9853538358476933e-05, + "loss": -0.0049, + "num_tokens": 2962550.0, + "reward": 0.8058516383171082, + "reward_std": 0.26573851704597473, + "rewards/cross_solution_unittest_reward/mean": 0.4029258117079735, + "rewards/cross_solution_unittest_reward/std": 0.2175045683979988, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 7 @@ -219,24 +219,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1060.0, - "completions/max_terminated_length": 1060.0, - "completions/mean_length": 527.5234375, - "completions/mean_terminated_length": 527.5234375, - "completions/min_length": 246.0, - "completions/min_terminated_length": 246.0, - "entropy": 0.074920654296875, + "completions/max_length": 1891.0, + "completions/max_terminated_length": 1891.0, + "completions/mean_length": 522.03125, + "completions/mean_terminated_length": 522.03125, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.1124114990234375, "epoch": 0.0729968634160251, "frac_reward_zero_std": 0.0, - "grad_norm": 0.025767840445041656, - "kl": 0.0002295374870300293, - "learning_rate": 1.2727272727272728e-05, - "loss": -0.0017, - "num_tokens": 3409458.0, - "reward": 0.546064019203186, - "reward_std": 0.06948773562908173, - "rewards/cross_solution_unittest/mean": 0.5460640788078308, - "rewards/cross_solution_unittest/std": 0.11339057236909866, + "grad_norm": 0.025463413447141647, + "kl": 0.00025027990341186523, + "learning_rate": 1.9800825610923937e-05, + "loss": 0.0018, + "num_tokens": 3417666.0, + "reward": 0.7429351210594177, + "reward_std": 0.21776002645492554, + "rewards/cross_solution_unittest_reward/mean": 0.37146755307912827, + "rewards/cross_solution_unittest_reward/std": 0.2737540230154991, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 8 @@ -247,25 +247,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1088.0, - "completions/max_terminated_length": 1088.0, - "completions/mean_length": 493.30731201171875, - "completions/mean_terminated_length": 493.30731201171875, - "completions/min_length": 214.0, - "completions/min_terminated_length": 214.0, - "entropy": 0.0809478759765625, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1064.0, + "completions/mean_length": 495.703125, + "completions/mean_terminated_length": 487.6396789550781, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.12152099609375, "epoch": 0.08212147134302823, "frac_reward_zero_std": 0.0, - "grad_norm": 0.026688260957598686, - "kl": 0.00023829936981201172, - "learning_rate": 1.4545454545454546e-05, - "loss": -0.0055, - "num_tokens": 3802052.0, - "reward": 0.5806035399436951, - "reward_std": 0.07414089143276215, - "rewards/cross_solution_unittest/mean": 0.5806035399436951, - "rewards/cross_solution_unittest/std": 0.09448928385972977, + "grad_norm": 0.02767878770828247, + "kl": 0.0002829134464263916, + "learning_rate": 1.9740119169423337e-05, + "loss": -0.0138, + "num_tokens": 3811456.0, + "reward": 0.8020304441452026, + "reward_std": 0.24267295002937317, + "rewards/cross_solution_unittest_reward/mean": 0.40101519227027893, + "rewards/cross_solution_unittest_reward/std": 0.21552012115716934, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 9 @@ -277,24 +277,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 939.0, - "completions/max_terminated_length": 939.0, - "completions/mean_length": 515.5573120117188, - "completions/mean_terminated_length": 515.5573120117188, - "completions/min_length": 277.0, - "completions/min_terminated_length": 277.0, - "entropy": 0.0720977783203125, + "completions/max_length": 994.0, + "completions/max_terminated_length": 994.0, + "completions/mean_length": 502.54168701171875, + "completions/mean_terminated_length": 502.54168701171875, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.1065216064453125, "epoch": 0.09124607927003137, "frac_reward_zero_std": 0.0, - "grad_norm": 0.02697775699198246, - "kl": 0.00022214651107788086, - "learning_rate": 1.6363636363636366e-05, - "loss": -0.0039, - "num_tokens": 4250610.0, - "reward": 0.6249701380729675, - "reward_std": 0.07456190884113312, - "rewards/cross_solution_unittest/mean": 0.6249701380729675, - "rewards/cross_solution_unittest/std": 0.11200571060180664, + "grad_norm": 0.029218332841992378, + "kl": 0.0002897977828979492, + "learning_rate": 1.9671468547019575e-05, + "loss": 0.0047, + "num_tokens": 4254672.0, + "reward": 0.8215359449386597, + "reward_std": 0.20994165539741516, + "rewards/cross_solution_unittest_reward/mean": 0.41076793521642685, + "rewards/cross_solution_unittest_reward/std": 0.2297622635960579, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 10 @@ -305,25 +305,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1229.0, - "completions/mean_length": 561.9869995117188, - "completions/mean_terminated_length": 554.0966186523438, - "completions/min_length": 296.0, - "completions/min_terminated_length": 296.0, - "entropy": 0.0686187744140625, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1138.0, + "completions/max_terminated_length": 1138.0, + "completions/mean_length": 546.2109375, + "completions/mean_terminated_length": 546.2109375, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.1016082763671875, "epoch": 0.1003706871970345, "frac_reward_zero_std": 0.0, - "grad_norm": 0.023875204846262932, - "kl": 0.00022393465042114258, - "learning_rate": 1.8181818181818182e-05, - "loss": -0.0013, - "num_tokens": 4686669.0, - "reward": 0.5655300617218018, - "reward_std": 0.08550869673490524, - "rewards/cross_solution_unittest/mean": 0.5655300617218018, - "rewards/cross_solution_unittest/std": 0.12360301613807678, + "grad_norm": 0.02588176354765892, + "kl": 0.00028312206268310547, + "learning_rate": 1.9594929736144978e-05, + "loss": 0.0049, + "num_tokens": 4680241.0, + "reward": 0.829417884349823, + "reward_std": 0.19322270154953003, + "rewards/cross_solution_unittest_reward/mean": 0.4147089347243309, + "rewards/cross_solution_unittest_reward/std": 0.20234351605176926, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 11 @@ -334,25 +334,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 995.0, - "completions/max_terminated_length": 995.0, - "completions/mean_length": 500.07293701171875, - "completions/mean_terminated_length": 500.07293701171875, - "completions/min_length": 213.0, - "completions/min_terminated_length": 213.0, - "entropy": 0.0841522216796875, + "completions/clipped_ratio": -4.96875, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 907.0, + "completions/mean_length": 506.40625, + "completions/mean_terminated_length": 490.293212890625, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.1226959228515625, "epoch": 0.10949529512403763, "frac_reward_zero_std": 0.0, - "grad_norm": 0.031900256872177124, - "kl": 0.0002987980842590332, - "learning_rate": 2e-05, - "loss": 0.0048, - "num_tokens": 5114985.0, - "reward": 0.628701388835907, - "reward_std": 0.0890752300620079, - "rewards/cross_solution_unittest/mean": 0.628701388835907, - "rewards/cross_solution_unittest/std": 0.13637541234493256, + "grad_norm": 0.03448358178138733, + "kl": 0.00037539005279541016, + "learning_rate": 1.9510565162951538e-05, + "loss": -0.0022, + "num_tokens": 5112749.0, + "reward": 0.6837499141693115, + "reward_std": 0.23883536458015442, + "rewards/cross_solution_unittest_reward/mean": 0.34187494963407516, + "rewards/cross_solution_unittest_reward/std": 0.24100807309150696, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 12 @@ -364,24 +364,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1188.0, - "completions/max_terminated_length": 1188.0, - "completions/mean_length": 525.2890625, - "completions/mean_terminated_length": 525.2890625, - "completions/min_length": 243.0, - "completions/min_terminated_length": 243.0, - "entropy": 0.07344818115234375, + "completions/max_length": 1035.0, + "completions/max_terminated_length": 1035.0, + "completions/mean_length": 534.4818115234375, + "completions/mean_terminated_length": 534.4818115234375, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "entropy": 0.1075286865234375, "epoch": 0.11861990305104078, "frac_reward_zero_std": 0.0, - "grad_norm": 0.0264806617051363, - "kl": 0.0002524852752685547, - "learning_rate": 1.9994965423831853e-05, - "loss": 0.0017, - "num_tokens": 5521312.0, - "reward": 0.5788325071334839, - "reward_std": 0.06292694807052612, - "rewards/cross_solution_unittest/mean": 0.5788324475288391, - "rewards/cross_solution_unittest/std": 0.10130160301923752, + "grad_norm": 0.027723070234060287, + "kl": 0.0003420412540435791, + "learning_rate": 1.941844363639525e-05, + "loss": -0.0135, + "num_tokens": 5525534.0, + "reward": 0.8237327337265015, + "reward_std": 0.19265763461589813, + "rewards/cross_solution_unittest_reward/mean": 0.41186633706092834, + "rewards/cross_solution_unittest_reward/std": 0.24469294399023056, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 13 @@ -392,25 +392,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.859375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 2440.0, - "completions/mean_length": 662.984375, - "completions/mean_terminated_length": 592.8800048828125, - "completions/min_length": 280.0, - "completions/min_terminated_length": 280.0, - "entropy": 0.07053375244140625, + "completions/clipped_ratio": -5.0, + "completions/max_length": 2351.0, + "completions/max_terminated_length": 2351.0, + "completions/mean_length": 615.5104370117188, + "completions/mean_terminated_length": 615.5104370117188, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.1039581298828125, "epoch": 0.1277445109780439, "frac_reward_zero_std": 0.0, - "grad_norm": 0.025386298075318336, - "kl": 0.00023430585861206055, - "learning_rate": 1.9979866764718846e-05, - "loss": -0.0055, - "num_tokens": 6009574.0, - "reward": 0.5900708436965942, - "reward_std": 0.0674651488661766, - "rewards/cross_solution_unittest/mean": 0.5900708436965942, - "rewards/cross_solution_unittest/std": 0.10158919543027878, + "grad_norm": 0.025678865611553192, + "kl": 0.00036969780921936035, + "learning_rate": 1.9318640292114526e-05, + "loss": -0.0176, + "num_tokens": 5993994.0, + "reward": 0.8750783205032349, + "reward_std": 0.16023778915405273, + "rewards/cross_solution_unittest_reward/mean": 0.43753916025161743, + "rewards/cross_solution_unittest_reward/std": 0.25390665233135223, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 14 @@ -421,25 +421,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, + "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 830.0, - "completions/mean_length": 475.04168701171875, - "completions/mean_terminated_length": 458.764404296875, - "completions/min_length": 254.0, - "completions/min_terminated_length": 254.0, - "entropy": 0.0851287841796875, + "completions/max_terminated_length": 826.0, + "completions/mean_length": 467.890625, + "completions/mean_terminated_length": 459.75457763671875, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "entropy": 0.122222900390625, "epoch": 0.13686911890504705, "frac_reward_zero_std": 0.0, - "grad_norm": 0.02971607819199562, - "kl": 0.00028899312019348145, - "learning_rate": 1.9954719225730847e-05, - "loss": 0.0129, - "num_tokens": 6357330.0, - "reward": 0.5344895124435425, - "reward_std": 0.08001528680324554, - "rewards/cross_solution_unittest/mean": 0.5344895720481873, - "rewards/cross_solution_unittest/std": 0.13401487469673157, + "grad_norm": 0.030205531045794487, + "kl": 0.00045168399810791016, + "learning_rate": 1.92112365311485e-05, + "loss": 0.0082, + "num_tokens": 6339752.0, + "reward": 0.8696179389953613, + "reward_std": 0.2131870836019516, + "rewards/cross_solution_unittest_reward/mean": 0.43480896949768066, + "rewards/cross_solution_unittest_reward/std": 0.2413550242781639, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 15 @@ -450,25 +450,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 3316.0, - "completions/mean_length": 538.6823120117188, - "completions/mean_terminated_length": 530.7310791015625, - "completions/min_length": 224.0, - "completions/min_terminated_length": 224.0, - "entropy": 0.0736846923828125, + "completions/clipped_ratio": -5.0, + "completions/max_length": 3274.0, + "completions/max_terminated_length": 3274.0, + "completions/mean_length": 536.6796875, + "completions/mean_terminated_length": 536.6796875, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.1075592041015625, "epoch": 0.1459937268320502, "frac_reward_zero_std": 0.0, - "grad_norm": 0.027988266199827194, - "kl": 0.00025466084480285645, - "learning_rate": 1.9919548128307954e-05, - "loss": -0.008, - "num_tokens": 6763168.0, - "reward": 0.5849003195762634, - "reward_std": 0.04933098703622818, - "rewards/cross_solution_unittest/mean": 0.5849003195762634, - "rewards/cross_solution_unittest/std": 0.1075245663523674, + "grad_norm": 0.027554774656891823, + "kl": 0.0004489421844482422, + "learning_rate": 1.9096319953545186e-05, + "loss": 0.001, + "num_tokens": 6744981.0, + "reward": 0.8156441450119019, + "reward_std": 0.22033701837062836, + "rewards/cross_solution_unittest_reward/mean": 0.40782201290130615, + "rewards/cross_solution_unittest_reward/std": 0.19903528690338135, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 16 @@ -479,25 +479,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 2246.0, - "completions/max_terminated_length": 2246.0, - "completions/mean_length": 522.3177490234375, - "completions/mean_terminated_length": 522.3177490234375, - "completions/min_length": 296.0, - "completions/min_terminated_length": 296.0, - "entropy": 0.078094482421875, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1317.0, + "completions/mean_length": 522.8880615234375, + "completions/mean_terminated_length": 514.8955688476562, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.1123504638671875, "epoch": 0.15511833475905332, "frac_reward_zero_std": 0.0, - "grad_norm": 0.025351958349347115, - "kl": 0.00026172399520874023, - "learning_rate": 1.9874388886763944e-05, - "loss": -0.0062, - "num_tokens": 7176030.0, - "reward": 0.5198549032211304, - "reward_std": 0.07214348018169403, - "rewards/cross_solution_unittest/mean": 0.5198548436164856, - "rewards/cross_solution_unittest/std": 0.14567282795906067, + "grad_norm": 0.027508998289704323, + "kl": 0.0005013346672058105, + "learning_rate": 1.8973984286913584e-05, + "loss": 0.0082, + "num_tokens": 7159690.0, + "reward": 0.9808885455131531, + "reward_std": 0.23347638547420502, + "rewards/cross_solution_unittest_reward/mean": 0.49044425785541534, + "rewards/cross_solution_unittest_reward/std": 0.28279364109039307, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 17 @@ -509,24 +509,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1374.0, - "completions/max_terminated_length": 1374.0, - "completions/mean_length": 536.2578125, - "completions/mean_terminated_length": 536.2578125, - "completions/min_length": 233.0, - "completions/min_terminated_length": 233.0, - "entropy": 0.07436370849609375, + "completions/max_length": 1142.0, + "completions/max_terminated_length": 1142.0, + "completions/mean_length": 522.3984375, + "completions/mean_terminated_length": 522.3984375, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.1051788330078125, "epoch": 0.16424294268605646, "frac_reward_zero_std": 0.0, - "grad_norm": 0.027073806151747704, - "kl": 0.0002745389938354492, - "learning_rate": 1.9819286972627066e-05, - "loss": 0.0116, - "num_tokens": 7612777.0, - "reward": 0.6148377656936646, - "reward_std": 0.05920207127928734, - "rewards/cross_solution_unittest/mean": 0.6148377060890198, - "rewards/cross_solution_unittest/std": 0.10591181367635727, + "grad_norm": 0.027137286961078644, + "kl": 0.0005218386650085449, + "learning_rate": 1.8844329309978146e-05, + "loss": -0.0001, + "num_tokens": 7590731.0, + "reward": 0.8454278707504272, + "reward_std": 0.19084712862968445, + "rewards/cross_solution_unittest_reward/mean": 0.42271392047405243, + "rewards/cross_solution_unittest_reward/std": 0.22437505424022675, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 18 @@ -539,23 +539,23 @@ "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1357.0, - "completions/mean_length": 507.0546875, - "completions/mean_terminated_length": 499.0209045410156, - "completions/min_length": 234.0, - "completions/min_terminated_length": 234.0, - "entropy": 0.0801239013671875, + "completions/max_terminated_length": 928.0, + "completions/mean_length": 507.03125, + "completions/mean_terminated_length": 498.9974060058594, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "entropy": 0.1159210205078125, "epoch": 0.17336755061305958, "frac_reward_zero_std": 0.0, - "grad_norm": 0.027948658913373947, - "kl": 0.00029531121253967285, - "learning_rate": 1.9754297868854075e-05, - "loss": -0.0075, - "num_tokens": 8027914.0, - "reward": 0.5844996571540833, - "reward_std": 0.07097827643156052, - "rewards/cross_solution_unittest/mean": 0.5844996571540833, - "rewards/cross_solution_unittest/std": 0.09445778280496597, + "grad_norm": 0.03020695596933365, + "kl": 0.0006084442138671875, + "learning_rate": 1.8707460771197773e-05, + "loss": 0.006, + "num_tokens": 8003919.0, + "reward": 0.7663086652755737, + "reward_std": 0.24537038803100586, + "rewards/cross_solution_unittest_reward/mean": 0.38315432518720627, + "rewards/cross_solution_unittest_reward/std": 0.2557295635342598, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 19 @@ -566,25 +566,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1284.0, - "completions/mean_length": 542.0234375, - "completions/mean_terminated_length": 534.0809326171875, - "completions/min_length": 298.0, - "completions/min_terminated_length": 298.0, - "entropy": 0.07269287109375, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1019.0, + "completions/max_terminated_length": 1019.0, + "completions/mean_length": 534.296875, + "completions/mean_terminated_length": 534.296875, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.1091766357421875, "epoch": 0.18249215854006273, "frac_reward_zero_std": 0.0, - "grad_norm": 0.028256384655833244, - "kl": 0.0003152787685394287, - "learning_rate": 1.9679487013963566e-05, - "loss": 0.0065, - "num_tokens": 8484811.0, - "reward": 0.6114683151245117, - "reward_std": 0.06493355333805084, - "rewards/cross_solution_unittest/mean": 0.6114682555198669, - "rewards/cross_solution_unittest/std": 0.09383873641490936, + "grad_norm": 0.03100455552339554, + "kl": 0.0006772875785827637, + "learning_rate": 1.856349030251589e-05, + "loss": -0.0103, + "num_tokens": 8459457.0, + "reward": 0.7904119491577148, + "reward_std": 0.23983600735664368, + "rewards/cross_solution_unittest_reward/mean": 0.39520594477653503, + "rewards/cross_solution_unittest_reward/std": 0.23401054739952087, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 20 @@ -596,24 +596,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 989.0, - "completions/max_terminated_length": 989.0, - "completions/mean_length": 483.76043701171875, - "completions/mean_terminated_length": 483.76043701171875, - "completions/min_length": 197.0, - "completions/min_terminated_length": 197.0, - "entropy": 0.0761260986328125, + "completions/max_length": 846.0, + "completions/max_terminated_length": 846.0, + "completions/mean_length": 485.5390625, + "completions/mean_terminated_length": 485.5390625, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.10430908203125, "epoch": 0.19161676646706588, "frac_reward_zero_std": 0.0, - "grad_norm": 0.027799971401691437, - "kl": 0.00030609965324401855, - "learning_rate": 1.9594929736144978e-05, - "loss": 0.0025, - "num_tokens": 8869807.0, - "reward": 0.6294624209403992, - "reward_std": 0.06047224625945091, - "rewards/cross_solution_unittest/mean": 0.6294623613357544, - "rewards/cross_solution_unittest/std": 0.10432589054107666, + "grad_norm": 0.032105084508657455, + "kl": 0.0006991028785705566, + "learning_rate": 1.8412535328311813e-05, + "loss": -0.0052, + "num_tokens": 8844608.0, + "reward": 0.8521513938903809, + "reward_std": 0.21744942665100098, + "rewards/cross_solution_unittest_reward/mean": 0.4260757490992546, + "rewards/cross_solution_unittest_reward/std": 0.21132270246744156, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 21 @@ -626,23 +626,23 @@ "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 951.0, - "completions/mean_length": 518.3724365234375, - "completions/mean_terminated_length": 510.3681640625, - "completions/min_length": 224.0, - "completions/min_terminated_length": 224.0, - "entropy": 0.07337188720703125, + "completions/max_terminated_length": 1558.0, + "completions/mean_length": 513.4479370117188, + "completions/mean_terminated_length": 505.4308166503906, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.1094970703125, "epoch": 0.200741374394069, "frac_reward_zero_std": 0.0, - "grad_norm": 0.029701806604862213, - "kl": 0.00030997395515441895, - "learning_rate": 1.9500711177409456e-05, - "loss": -0.0053, - "num_tokens": 9251294.0, - "reward": 0.581719160079956, - "reward_std": 0.07320186495780945, - "rewards/cross_solution_unittest/mean": 0.581719160079956, - "rewards/cross_solution_unittest/std": 0.16921019554138184, + "grad_norm": 0.029629480093717575, + "kl": 0.0006964206695556641, + "learning_rate": 1.825471896962774e-05, + "loss": -0.0126, + "num_tokens": 9223148.0, + "reward": 0.8136462569236755, + "reward_std": 0.21329909563064575, + "rewards/cross_solution_unittest_reward/mean": 0.4068231135606766, + "rewards/cross_solution_unittest_reward/std": 0.24406656622886658, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 22 @@ -655,23 +655,23 @@ "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1624.0, - "completions/mean_length": 529.96875, - "completions/mean_terminated_length": 521.9948120117188, - "completions/min_length": 272.0, - "completions/min_terminated_length": 272.0, - "entropy": 0.0753173828125, + "completions/max_terminated_length": 1091.0, + "completions/mean_length": 513.3724365234375, + "completions/mean_terminated_length": 505.3551025390625, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.1093292236328125, "epoch": 0.20986598232107215, "frac_reward_zero_std": 0.0, - "grad_norm": 0.027412766590714455, - "kl": 0.0003858804702758789, - "learning_rate": 1.9396926207859085e-05, - "loss": -0.0033, - "num_tokens": 9705346.0, - "reward": 0.5391039848327637, - "reward_std": 0.06848187744617462, - "rewards/cross_solution_unittest/mean": 0.5391039848327637, - "rewards/cross_solution_unittest/std": 0.1495925337076187, + "grad_norm": 0.028564205393195152, + "kl": 0.0008405447006225586, + "learning_rate": 1.8090169943749477e-05, + "loss": 0.0105, + "num_tokens": 9671827.0, + "reward": 0.9153262376785278, + "reward_std": 0.19782906770706177, + "rewards/cross_solution_unittest_reward/mean": 0.45766307413578033, + "rewards/cross_solution_unittest_reward/std": 0.23260264098644257, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 23 @@ -683,24 +683,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 895.0, - "completions/max_terminated_length": 895.0, - "completions/mean_length": 508.359375, - "completions/mean_terminated_length": 508.359375, - "completions/min_length": 171.0, - "completions/min_terminated_length": 171.0, - "entropy": 0.07342529296875, + "completions/max_length": 970.0, + "completions/max_terminated_length": 970.0, + "completions/mean_length": 499.43231201171875, + "completions/mean_terminated_length": 499.43231201171875, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.1064300537109375, "epoch": 0.21899059024807527, "frac_reward_zero_std": 0.0, - "grad_norm": 0.025877749547362328, - "kl": 0.00034868717193603516, - "learning_rate": 1.9283679330160726e-05, - "loss": 0.0015, - "num_tokens": 10126856.0, - "reward": 0.601729691028595, - "reward_std": 0.08979599177837372, - "rewards/cross_solution_unittest/mean": 0.6017296314239502, - "rewards/cross_solution_unittest/std": 0.13213877379894257, + "grad_norm": 0.02926481142640114, + "kl": 0.0008217096328735352, + "learning_rate": 1.7919022459222754e-05, + "loss": 0.0034, + "num_tokens": 10091281.0, + "reward": 0.7448365092277527, + "reward_std": 0.2386973798274994, + "rewards/cross_solution_unittest_reward/mean": 0.37241824716329575, + "rewards/cross_solution_unittest_reward/std": 0.2582271620631218, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 24 @@ -711,25 +711,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1476.0, - "completions/max_terminated_length": 1476.0, - "completions/mean_length": 500.75262451171875, - "completions/mean_terminated_length": 500.75262451171875, - "completions/min_length": 259.0, - "completions/min_terminated_length": 259.0, - "entropy": 0.0807342529296875, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1083.0, + "completions/mean_length": 496.18231201171875, + "completions/mean_terminated_length": 488.1201171875, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "entropy": 0.11749267578125, "epoch": 0.22811519817507842, "frac_reward_zero_std": 0.0, - "grad_norm": 0.030456064268946648, - "kl": 0.00040835142135620117, - "learning_rate": 1.9161084574320696e-05, - "loss": 0.0033, - "num_tokens": 10534033.0, - "reward": 0.6099452972412109, - "reward_std": 0.05933503434062004, - "rewards/cross_solution_unittest/mean": 0.6099452376365662, - "rewards/cross_solution_unittest/std": 0.08372820913791656, + "grad_norm": 0.03824177011847496, + "kl": 0.001001119613647461, + "learning_rate": 1.7741416106390828e-05, + "loss": 0.0011, + "num_tokens": 10495487.0, + "reward": 0.8449365496635437, + "reward_std": 0.20980952680110931, + "rewards/cross_solution_unittest_reward/mean": 0.42246825993061066, + "rewards/cross_solution_unittest_reward/std": 0.21189714968204498, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 25 @@ -740,25 +740,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1958.0, - "completions/mean_length": 536.2213745117188, - "completions/mean_terminated_length": 520.264404296875, - "completions/min_length": 253.0, - "completions/min_terminated_length": 253.0, - "entropy": 0.077789306640625, + "completions/clipped_ratio": -5.0, + "completions/max_length": 848.0, + "completions/max_terminated_length": 848.0, + "completions/mean_length": 497.1458435058594, + "completions/mean_terminated_length": 497.1458435058594, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.11029052734375, "epoch": 0.23723980610208156, "frac_reward_zero_std": 0.0, - "grad_norm": 0.027375157922506332, - "kl": 0.0004361867904663086, - "learning_rate": 1.9029265382866216e-05, - "loss": -0.0071, - "num_tokens": 10909678.0, - "reward": 0.5840487480163574, - "reward_std": 0.07150473445653915, - "rewards/cross_solution_unittest/mean": 0.5840487480163574, - "rewards/cross_solution_unittest/std": 0.12009274214506149, + "grad_norm": 0.028777848929166794, + "kl": 0.0008819103240966797, + "learning_rate": 1.7557495743542586e-05, + "loss": -0.0032, + "num_tokens": 10853559.0, + "reward": 0.7521219253540039, + "reward_std": 0.18529602885246277, + "rewards/cross_solution_unittest_reward/mean": 0.37606097012758255, + "rewards/cross_solution_unittest_reward/std": 0.24145866930484772, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 26 @@ -770,24 +770,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 838.0, - "completions/max_terminated_length": 838.0, - "completions/mean_length": 471.58856201171875, - "completions/mean_terminated_length": 471.58856201171875, - "completions/min_length": 230.0, - "completions/min_terminated_length": 230.0, - "entropy": 0.0792694091796875, + "completions/max_length": 766.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 468.05731201171875, + "completions/mean_terminated_length": 468.05731201171875, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.1114501953125, "epoch": 0.24636441402908468, "frac_reward_zero_std": 0.0, - "grad_norm": 0.027460293844342232, - "kl": 0.000461578369140625, - "learning_rate": 1.8888354486549238e-05, - "loss": -0.0034, - "num_tokens": 11287180.0, - "reward": 0.5752553939819336, - "reward_std": 0.06591899693012238, - "rewards/cross_solution_unittest/mean": 0.5752554535865784, - "rewards/cross_solution_unittest/std": 0.12003585696220398, + "grad_norm": 0.02834751084446907, + "kl": 0.0009758472442626953, + "learning_rate": 1.736741137876405e-05, + "loss": -0.0084, + "num_tokens": 11230181.0, + "reward": 0.8723708987236023, + "reward_std": 0.19247429072856903, + "rewards/cross_solution_unittest_reward/mean": 0.4361855015158653, + "rewards/cross_solution_unittest_reward/std": 0.21063780039548874, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 27 @@ -798,25 +798,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1051.0, - "completions/max_terminated_length": 1051.0, - "completions/mean_length": 511.953125, - "completions/mean_terminated_length": 511.953125, - "completions/min_length": 184.0, - "completions/min_terminated_length": 184.0, - "entropy": 0.075927734375, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1015.0, + "completions/mean_length": 507.0989685058594, + "completions/mean_terminated_length": 499.0652770996094, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.1055908203125, "epoch": 0.2554890219560878, "frac_reward_zero_std": 0.0, - "grad_norm": 0.029021723195910454, - "kl": 0.0005039572715759277, - "learning_rate": 1.873849377069785e-05, - "loss": -0.0042, - "num_tokens": 11722918.0, - "reward": 0.5585534572601318, - "reward_std": 0.08278067409992218, - "rewards/cross_solution_unittest/mean": 0.5585535168647766, - "rewards/cross_solution_unittest/std": 0.15765663981437683, + "grad_norm": 0.029102731496095657, + "kl": 0.0011304616928100586, + "learning_rate": 1.7171318047589637e-05, + "loss": -0.0081, + "num_tokens": 11663099.0, + "reward": 0.9932507276535034, + "reward_std": 0.18286040425300598, + "rewards/cross_solution_unittest_reward/mean": 0.4966253191232681, + "rewards/cross_solution_unittest_reward/std": 0.2659049928188324, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 28 @@ -829,23 +829,23 @@ "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1088.0, - "completions/mean_length": 498.6302185058594, - "completions/mean_terminated_length": 490.5744323730469, - "completions/min_length": 220.0, - "completions/min_terminated_length": 220.0, - "entropy": 0.08233642578125, + "completions/max_terminated_length": 1139.0, + "completions/mean_length": 489.8515625, + "completions/mean_terminated_length": 481.7728576660156, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.113128662109375, "epoch": 0.26461362988309095, "frac_reward_zero_std": 0.0, - "grad_norm": 0.029287124052643776, - "kl": 0.0005979537963867188, - "learning_rate": 1.8579834132349773e-05, - "loss": 0.016, - "num_tokens": 12122528.0, - "reward": 0.5899044275283813, - "reward_std": 0.060223691165447235, - "rewards/cross_solution_unittest/mean": 0.5899044871330261, - "rewards/cross_solution_unittest/std": 0.09450040012598038, + "grad_norm": 0.030337095260620117, + "kl": 0.0015281438827514648, + "learning_rate": 1.696937568655294e-05, + "loss": 0.0098, + "num_tokens": 12059194.0, + "reward": 0.8777666091918945, + "reward_std": 0.2015705108642578, + "rewards/cross_solution_unittest_reward/mean": 0.43888331949710846, + "rewards/cross_solution_unittest_reward/std": 0.289621040225029, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 29 @@ -856,25 +856,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.953125, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 2053.0, - "completions/mean_length": 553.2057495117188, - "completions/mean_terminated_length": 529.3411865234375, - "completions/min_length": 261.0, - "completions/min_terminated_length": 261.0, - "entropy": 0.075469970703125, + "completions/clipped_ratio": -5.0, + "completions/max_length": 863.0, + "completions/max_terminated_length": 863.0, + "completions/mean_length": 518.9349365234375, + "completions/mean_terminated_length": 518.9349365234375, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.10247802734375, "epoch": 0.2737382378100941, "frac_reward_zero_std": 0.0, - "grad_norm": 0.028479870408773422, - "kl": 0.0005797147750854492, - "learning_rate": 1.8412535328311813e-05, - "loss": 0.0051, - "num_tokens": 12555555.0, - "reward": 0.5730323791503906, - "reward_std": 0.11489957571029663, - "rewards/cross_solution_unittest/mean": 0.5730323195457458, - "rewards/cross_solution_unittest/std": 0.18645499646663666, + "grad_norm": 0.028208184987306595, + "kl": 0.001240849494934082, + "learning_rate": 1.6761749002740195e-05, + "loss": -0.0072, + "num_tokens": 12477345.0, + "reward": 0.7518314123153687, + "reward_std": 0.223491370677948, + "rewards/cross_solution_unittest_reward/mean": 0.37591565400362015, + "rewards/cross_solution_unittest_reward/std": 0.2666970565915108, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 30 @@ -885,25 +885,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 972.0, - "completions/max_terminated_length": 972.0, - "completions/mean_length": 473.4453125, - "completions/mean_terminated_length": 473.4453125, - "completions/min_length": 217.0, - "completions/min_terminated_length": 217.0, - "entropy": 0.084716796875, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 470.55731201171875, + "completions/mean_terminated_length": 462.4281921386719, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.10906982421875, "epoch": 0.28286284573709725, "frac_reward_zero_std": 0.0, - "grad_norm": 0.02932862937450409, - "kl": 0.0007239580154418945, - "learning_rate": 1.8236765814298328e-05, - "loss": 0.0069, - "num_tokens": 12922190.0, - "reward": 0.6376906633377075, - "reward_std": 0.07534094899892807, - "rewards/cross_solution_unittest/mean": 0.6376906037330627, - "rewards/cross_solution_unittest/std": 0.11291607469320297, + "grad_norm": 0.03271934762597084, + "kl": 0.0014473199844360352, + "learning_rate": 1.6548607339452853e-05, + "loss": 0.0095, + "num_tokens": 12841471.0, + "reward": 0.9323596954345703, + "reward_std": 0.1906483769416809, + "rewards/cross_solution_unittest_reward/mean": 0.46617985516786575, + "rewards/cross_solution_unittest_reward/std": 0.21004272997379303, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 31 @@ -914,25 +914,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1047.0, - "completions/max_terminated_length": 1047.0, - "completions/mean_length": 485.9609375, - "completions/mean_terminated_length": 485.9609375, - "completions/min_length": 226.0, - "completions/min_terminated_length": 226.0, - "entropy": 0.0768280029296875, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 906.0, + "completions/mean_length": 483.4505310058594, + "completions/mean_terminated_length": 475.3551025390625, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.105499267578125, "epoch": 0.2919874536641004, "frac_reward_zero_std": 0.0, - "grad_norm": 0.0313863605260849, - "kl": 0.000737309455871582, - "learning_rate": 1.8052702575310588e-05, - "loss": -0.0043, - "num_tokens": 13378943.0, - "reward": 0.6212249994277954, - "reward_std": 0.08826343715190887, - "rewards/cross_solution_unittest/mean": 0.6212249398231506, - "rewards/cross_solution_unittest/std": 0.1459883153438568, + "grad_norm": 0.045950278639793396, + "kl": 0.00164031982421875, + "learning_rate": 1.6330124538088705e-05, + "loss": 0.0251, + "num_tokens": 13295500.0, + "reward": 0.9082147479057312, + "reward_std": 0.3082047402858734, + "rewards/cross_solution_unittest_reward/mean": 0.4541073888540268, + "rewards/cross_solution_unittest_reward/std": 0.24801740050315857, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 32 @@ -945,23 +945,23 @@ "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1008.0, - "completions/mean_length": 500.5911560058594, - "completions/mean_terminated_length": 492.54046630859375, - "completions/min_length": 225.0, - "completions/min_terminated_length": 225.0, - "entropy": 0.0904998779296875, + "completions/max_terminated_length": 778.0, + "completions/mean_length": 476.5130310058594, + "completions/mean_terminated_length": 468.39947509765625, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "entropy": 0.11260986328125, "epoch": 0.3011120615911035, "frac_reward_zero_std": 0.0, - "grad_norm": 0.028566641733050346, - "kl": 0.0008232593536376953, - "learning_rate": 1.7860530947427878e-05, - "loss": -0.0054, - "num_tokens": 13765758.0, - "reward": 0.6300634741783142, - "reward_std": 0.07626457512378693, - "rewards/cross_solution_unittest/mean": 0.6300634741783142, - "rewards/cross_solution_unittest/std": 0.11189531534910202, + "grad_norm": 0.03122815489768982, + "kl": 0.001780867576599121, + "learning_rate": 1.6106478796354382e-05, + "loss": 0.0068, + "num_tokens": 13676457.0, + "reward": 0.8194860816001892, + "reward_std": 0.1633000671863556, + "rewards/cross_solution_unittest_reward/mean": 0.4097430929541588, + "rewards/cross_solution_unittest_reward/std": 0.2260184958577156, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 33 @@ -973,24 +973,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 931.0, - "completions/max_terminated_length": 931.0, - "completions/mean_length": 505.2109375, - "completions/mean_terminated_length": 505.2109375, - "completions/min_length": 254.0, - "completions/min_terminated_length": 254.0, - "entropy": 0.0770263671875, + "completions/max_length": 794.0, + "completions/max_terminated_length": 794.0, + "completions/mean_length": 480.32293701171875, + "completions/mean_terminated_length": 480.32293701171875, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.10528564453125, "epoch": 0.31023666951810663, "frac_reward_zero_std": 0.0, - "grad_norm": 0.02723107673227787, - "kl": 0.0009553432464599609, - "learning_rate": 1.766044443118978e-05, - "loss": 0.0036, - "num_tokens": 14200039.0, - "reward": 0.6032774448394775, - "reward_std": 0.05867696925997734, - "rewards/cross_solution_unittest/mean": 0.6032774448394775, - "rewards/cross_solution_unittest/std": 0.0986337959766388, + "grad_norm": 0.03177294880151749, + "kl": 0.002082347869873047, + "learning_rate": 1.5877852522924733e-05, + "loss": -0.0068, + "num_tokens": 14100165.0, + "reward": 0.9499843120574951, + "reward_std": 0.18478095531463623, + "rewards/cross_solution_unittest_reward/mean": 0.47499214112758636, + "rewards/cross_solution_unittest_reward/std": 0.23216352611780167, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 34 @@ -1001,25 +1001,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 2461.0, - "completions/mean_length": 516.0833740234375, - "completions/mean_terminated_length": 508.0731201171875, - "completions/min_length": 231.0, - "completions/min_terminated_length": 231.0, - "entropy": 0.07989501953125, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1548.0, + "completions/max_terminated_length": 1548.0, + "completions/mean_length": 490.5208435058594, + "completions/mean_terminated_length": 490.5208435058594, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.109344482421875, "epoch": 0.3193612774451098, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03081453964114189, - "kl": 0.0009756088256835938, - "learning_rate": 1.745264449675755e-05, - "loss": -0.0036, - "num_tokens": 14588483.0, - "reward": 0.5146794319152832, - "reward_std": 0.07540898025035858, - "rewards/cross_solution_unittest/mean": 0.514679491519928, - "rewards/cross_solution_unittest/std": 0.1279023289680481, + "grad_norm": 0.03219935670495033, + "kl": 0.001997709274291992, + "learning_rate": 1.5644432188667695e-05, + "loss": 0.0101, + "num_tokens": 14479845.0, + "reward": 0.8249267339706421, + "reward_std": 0.22756871581077576, + "rewards/cross_solution_unittest_reward/mean": 0.41246332228183746, + "rewards/cross_solution_unittest_reward/std": 0.249651201069355, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 35 @@ -1031,24 +1031,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 947.0, - "completions/max_terminated_length": 947.0, - "completions/mean_length": 523.4557495117188, - "completions/mean_terminated_length": 523.4557495117188, - "completions/min_length": 319.0, - "completions/min_terminated_length": 319.0, - "entropy": 0.079742431640625, + "completions/max_length": 1074.0, + "completions/max_terminated_length": 1074.0, + "completions/mean_length": 508.5546875, + "completions/mean_terminated_length": 508.5546875, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.110809326171875, "epoch": 0.32848588537211293, "frac_reward_zero_std": 0.0, - "grad_norm": 0.0302255991846323, - "kl": 0.0010718107223510742, - "learning_rate": 1.72373403810507e-05, - "loss": 0.0038, - "num_tokens": 15012890.0, - "reward": 0.5417085886001587, - "reward_std": 0.07886814326047897, - "rewards/cross_solution_unittest/mean": 0.5417085289955139, - "rewards/cross_solution_unittest/std": 0.14164547622203827, + "grad_norm": 0.033212192356586456, + "kl": 0.0023381710052490234, + "learning_rate": 1.5406408174555978e-05, + "loss": -0.007, + "num_tokens": 14897138.0, + "reward": 0.7724038362503052, + "reward_std": 0.2212439626455307, + "rewards/cross_solution_unittest_reward/mean": 0.3862019032239914, + "rewards/cross_solution_unittest_reward/std": 0.2340112403035164, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 36 @@ -1060,24 +1060,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1876.0, - "completions/max_terminated_length": 1876.0, - "completions/mean_length": 514.140625, - "completions/mean_terminated_length": 514.140625, - "completions/min_length": 227.0, - "completions/min_terminated_length": 227.0, - "entropy": 0.0728759765625, + "completions/max_length": 957.0, + "completions/max_terminated_length": 957.0, + "completions/mean_length": 499.2239685058594, + "completions/mean_terminated_length": 499.2239685058594, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.0943603515625, "epoch": 0.3376104932991161, "frac_reward_zero_std": 0.0, - "grad_norm": 0.030379414558410645, - "kl": 0.0013577938079833984, - "learning_rate": 1.7014748877063212e-05, - "loss": -0.0051, - "num_tokens": 15419892.0, - "reward": 0.6429738402366638, - "reward_std": 0.07094022631645203, - "rewards/cross_solution_unittest/mean": 0.6429738402366638, - "rewards/cross_solution_unittest/std": 0.1320594996213913, + "grad_norm": 0.032685279846191406, + "kl": 0.0021796226501464844, + "learning_rate": 1.5163974616389621e-05, + "loss": -0.004, + "num_tokens": 15299728.0, + "reward": 0.8877171874046326, + "reward_std": 0.1671483963727951, + "rewards/cross_solution_unittest_reward/mean": 0.4438585937023163, + "rewards/cross_solution_unittest_reward/std": 0.2307201847434044, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 37 @@ -1089,24 +1089,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 887.0, - "completions/max_terminated_length": 887.0, - "completions/mean_length": 477.8046875, - "completions/mean_terminated_length": 477.8046875, - "completions/min_length": 231.0, - "completions/min_terminated_length": 231.0, - "entropy": 0.079803466796875, + "completions/max_length": 1086.0, + "completions/max_terminated_length": 1086.0, + "completions/mean_length": 479.34375, + "completions/mean_terminated_length": 479.34375, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "entropy": 0.1061859130859375, "epoch": 0.34673510122611917, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03055444173514843, - "kl": 0.0011442899703979492, - "learning_rate": 1.6785094115571323e-05, - "loss": -0.0004, - "num_tokens": 15801089.0, - "reward": 0.5747675895690918, - "reward_std": 0.06717974692583084, - "rewards/cross_solution_unittest/mean": 0.5747675895690918, - "rewards/cross_solution_unittest/std": 0.08518695831298828, + "grad_norm": 0.031533192843198776, + "kl": 0.0025458335876464844, + "learning_rate": 1.491732924645604e-05, + "loss": 0.0057, + "num_tokens": 15681980.0, + "reward": 0.8841136693954468, + "reward_std": 0.18882207572460175, + "rewards/cross_solution_unittest_reward/mean": 0.442056804895401, + "rewards/cross_solution_unittest_reward/std": 0.23417016118764877, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 38 @@ -1119,23 +1119,23 @@ "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1270.0, - "completions/mean_length": 476.1770935058594, - "completions/mean_terminated_length": 468.06268310546875, - "completions/min_length": 131.0, - "completions/min_terminated_length": 131.0, - "entropy": 0.087554931640625, + "completions/max_terminated_length": 975.0, + "completions/mean_length": 462.5520935058594, + "completions/mean_terminated_length": 454.402099609375, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "entropy": 0.11846923828125, "epoch": 0.3558597091531223, "frac_reward_zero_std": 0.0, - "grad_norm": 0.04223545268177986, - "kl": 0.0014077425003051758, - "learning_rate": 1.6548607339452853e-05, - "loss": 0.0023, - "num_tokens": 16114473.0, - "reward": 0.5905721187591553, - "reward_std": 0.09474077075719833, - "rewards/cross_solution_unittest/mean": 0.5905721187591553, - "rewards/cross_solution_unittest/std": 0.13974814116954803, + "grad_norm": 0.09556223452091217, + "kl": 0.0063495635986328125, + "learning_rate": 1.4666673232256738e-05, + "loss": 0.0078, + "num_tokens": 15989736.0, + "reward": 0.7538800239562988, + "reward_std": 0.2889000177383423, + "rewards/cross_solution_unittest_reward/mean": 0.3769400045275688, + "rewards/cross_solution_unittest_reward/std": 0.2265937775373459, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 39 @@ -1146,25 +1146,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, + "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 987.0, - "completions/mean_length": 491.4895935058594, - "completions/mean_terminated_length": 475.2984313964844, - "completions/min_length": 232.0, - "completions/min_terminated_length": 232.0, - "entropy": 0.07975006103515625, + "completions/max_terminated_length": 1821.0, + "completions/mean_length": 492.40625, + "completions/mean_terminated_length": 484.3341979980469, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.1049957275390625, "epoch": 0.36498431708012546, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03678172081708908, - "kl": 0.0016982555389404297, - "learning_rate": 1.6305526670845225e-05, - "loss": -0.0074, - "num_tokens": 16524457.0, - "reward": 0.5999099612236023, - "reward_std": 0.09236396849155426, - "rewards/cross_solution_unittest/mean": 0.5999099612236023, - "rewards/cross_solution_unittest/std": 0.1273658722639084, + "grad_norm": 0.10226094722747803, + "kl": 0.006191730499267578, + "learning_rate": 1.4412211012432213e-05, + "loss": -0.0045, + "num_tokens": 16399484.0, + "reward": 0.7214807271957397, + "reward_std": 0.20936036109924316, + "rewards/cross_solution_unittest_reward/mean": 0.3607403114438057, + "rewards/cross_solution_unittest_reward/std": 0.286220021545887, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 40 @@ -1175,25 +1175,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, + "completions/clipped_ratio": -4.953125, "completions/max_length": 3584.0, - "completions/max_terminated_length": 924.0, - "completions/mean_length": 501.82293701171875, - "completions/mean_terminated_length": 493.77545166015625, - "completions/min_length": 234.0, - "completions/min_terminated_length": 234.0, - "entropy": 0.08660888671875, + "completions/max_terminated_length": 1305.0, + "completions/mean_length": 521.59375, + "completions/mean_terminated_length": 497.4803161621094, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.1065673828125, "epoch": 0.3741089250071286, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03592362627387047, - "kl": 0.0014951229095458984, - "learning_rate": 1.6056096871376667e-05, - "loss": -0.0111, - "num_tokens": 16940425.0, - "reward": 0.6201981902122498, - "reward_std": 0.07472098618745804, - "rewards/cross_solution_unittest/mean": 0.620198130607605, - "rewards/cross_solution_unittest/std": 0.12231894582509995, + "grad_norm": 0.03374212980270386, + "kl": 0.003753662109375, + "learning_rate": 1.4154150130018867e-05, + "loss": 0.0217, + "num_tokens": 16819632.0, + "reward": 0.6985855102539062, + "reward_std": 0.17247408628463745, + "rewards/cross_solution_unittest_reward/mean": 0.34929272532463074, + "rewards/cross_solution_unittest_reward/std": 0.21424289792776108, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 41 @@ -1204,25 +1204,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1250.0, - "completions/max_terminated_length": 1250.0, - "completions/mean_length": 526.3515625, - "completions/mean_terminated_length": 526.3515625, - "completions/min_length": 262.0, - "completions/min_terminated_length": 262.0, - "entropy": 0.069305419921875, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 777.0, + "completions/mean_length": 498.4192810058594, + "completions/mean_terminated_length": 490.3629455566406, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "entropy": 0.0918121337890625, "epoch": 0.38323353293413176, "frac_reward_zero_std": 0.0, - "grad_norm": 0.032890286296606064, - "kl": 0.0016560554504394531, - "learning_rate": 1.5800569095711983e-05, - "loss": -0.0149, - "num_tokens": 17392560.0, - "reward": 0.5883609652519226, - "reward_std": 0.09757032245397568, - "rewards/cross_solution_unittest/mean": 0.5883609056472778, - "rewards/cross_solution_unittest/std": 0.138299360871315, + "grad_norm": 0.0317884236574173, + "kl": 0.00311279296875, + "learning_rate": 1.3892701063173917e-05, + "loss": 0.0088, + "num_tokens": 17261449.0, + "reward": 0.8251139521598816, + "reward_std": 0.16213150322437286, + "rewards/cross_solution_unittest_reward/mean": 0.4125569462776184, + "rewards/cross_solution_unittest_reward/std": 0.23735850304365158, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 42 @@ -1233,25 +1233,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 883.0, - "completions/mean_length": 492.0703125, - "completions/mean_terminated_length": 483.9974060058594, - "completions/min_length": 253.0, - "completions/min_terminated_length": 253.0, - "entropy": 0.0814666748046875, + "completions/clipped_ratio": -5.0, + "completions/max_length": 758.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 464.19012451171875, + "completions/mean_terminated_length": 464.19012451171875, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.103759765625, "epoch": 0.39235814086113485, "frac_reward_zero_std": 0.0, - "grad_norm": 0.030933000147342682, - "kl": 0.0019317865371704102, - "learning_rate": 1.5539200638661106e-05, - "loss": 0.0112, - "num_tokens": 17737743.0, - "reward": 0.6191085577011108, - "reward_std": 0.07476722449064255, - "rewards/cross_solution_unittest/mean": 0.6191084980964661, - "rewards/cross_solution_unittest/std": 0.11829123646020889, + "grad_norm": 0.033187512308359146, + "kl": 0.0038928985595703125, + "learning_rate": 1.362807705350641e-05, + "loss": 0.0018, + "num_tokens": 17594050.0, + "reward": 0.9705511927604675, + "reward_std": 0.15792036056518555, + "rewards/cross_solution_unittest_reward/mean": 0.48527561128139496, + "rewards/cross_solution_unittest_reward/std": 0.25453296303749084, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 43 @@ -1264,23 +1264,23 @@ "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1413.0, - "completions/mean_length": 531.0286865234375, - "completions/mean_terminated_length": 523.0574340820312, - "completions/min_length": 247.0, - "completions/min_terminated_length": 247.0, - "entropy": 0.07323455810546875, + "completions/max_terminated_length": 1812.0, + "completions/mean_length": 511.6640625, + "completions/mean_terminated_length": 503.6423034667969, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.0930633544921875, "epoch": 0.401482748788138, "frac_reward_zero_std": 0.0, - "grad_norm": 0.02944750152528286, - "kl": 0.0017609596252441406, - "learning_rate": 1.5272254676105026e-05, - "loss": -0.0072, - "num_tokens": 18154030.0, - "reward": 0.5842992067337036, - "reward_std": 0.07545933127403259, - "rewards/cross_solution_unittest/mean": 0.5842992067337036, - "rewards/cross_solution_unittest/std": 0.13810177147388458, + "grad_norm": 0.02915688417851925, + "kl": 0.0028166770935058594, + "learning_rate": 1.3360493932154301e-05, + "loss": 0.0031, + "num_tokens": 18001433.0, + "reward": 0.6709164977073669, + "reward_std": 0.17882965505123138, + "rewards/cross_solution_unittest_reward/mean": 0.3354582153260708, + "rewards/cross_solution_unittest_reward/std": 0.20766809582710266, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 44 @@ -1291,25 +1291,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 938.0, - "completions/max_terminated_length": 938.0, - "completions/mean_length": 468.75, - "completions/mean_terminated_length": 468.75, - "completions/min_length": 224.0, - "completions/min_terminated_length": 224.0, - "entropy": 0.0874786376953125, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 855.0, + "completions/mean_length": 456.7265625, + "completions/mean_terminated_length": 448.5613708496094, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "entropy": 0.1132659912109375, "epoch": 0.41060735671514115, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03713710606098175, - "kl": 0.002506732940673828, - "learning_rate": 1.5000000000000002e-05, - "loss": 0.0025, - "num_tokens": 18535066.0, - "reward": 0.6071228981018066, - "reward_std": 0.08849634975194931, - "rewards/cross_solution_unittest/mean": 0.6071229577064514, - "rewards/cross_solution_unittest/std": 0.1443415731191635, + "grad_norm": 0.038488611578941345, + "kl": 0.004986763000488281, + "learning_rate": 1.3090169943749475e-05, + "loss": 0.0137, + "num_tokens": 18379368.0, + "reward": 0.9005228281021118, + "reward_std": 0.19103503227233887, + "rewards/cross_solution_unittest_reward/mean": 0.4502613842487335, + "rewards/cross_solution_unittest_reward/std": 0.25590047985315323, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 45 @@ -1321,24 +1321,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 885.0, - "completions/max_terminated_length": 885.0, - "completions/mean_length": 507.75262451171875, - "completions/mean_terminated_length": 507.75262451171875, - "completions/min_length": 277.0, - "completions/min_terminated_length": 277.0, - "entropy": 0.07917022705078125, + "completions/max_length": 3564.0, + "completions/max_terminated_length": 3564.0, + "completions/mean_length": 498.2630310058594, + "completions/mean_terminated_length": 498.2630310058594, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.098541259765625, "epoch": 0.4197319646421443, "frac_reward_zero_std": 0.0, - "grad_norm": 0.030308177694678307, - "kl": 0.0022301673889160156, - "learning_rate": 1.472271074772683e-05, - "loss": -0.0041, - "num_tokens": 18951727.0, - "reward": 0.6283596158027649, - "reward_std": 0.05247451364994049, - "rewards/cross_solution_unittest/mean": 0.6283596158027649, - "rewards/cross_solution_unittest/std": 0.07481828331947327, + "grad_norm": 0.034151654690504074, + "kl": 0.0035920143127441406, + "learning_rate": 1.2817325568414299e-05, + "loss": -0.0086, + "num_tokens": 18790933.0, + "reward": 0.7998638153076172, + "reward_std": 0.26316848397254944, + "rewards/cross_solution_unittest_reward/mean": 0.3999319151043892, + "rewards/cross_solution_unittest_reward/std": 0.2370620146393776, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 46 @@ -1350,24 +1350,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1472.0, - "completions/max_terminated_length": 1472.0, - "completions/mean_length": 498.56512451171875, - "completions/mean_terminated_length": 498.56512451171875, - "completions/min_length": 239.0, - "completions/min_terminated_length": 239.0, - "entropy": 0.0898895263671875, + "completions/max_length": 908.0, + "completions/max_terminated_length": 908.0, + "completions/mean_length": 479.18231201171875, + "completions/mean_terminated_length": 479.18231201171875, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.11773681640625, "epoch": 0.42885657256914744, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03285127133131027, - "kl": 0.0027070045471191406, - "learning_rate": 1.4440666126057743e-05, - "loss": -0.0049, - "num_tokens": 19368960.0, - "reward": 0.5909382104873657, - "reward_std": 0.07557514309883118, - "rewards/cross_solution_unittest/mean": 0.590938150882721, - "rewards/cross_solution_unittest/std": 0.14489254355430603, + "grad_norm": 0.037232693284749985, + "kl": 0.004299640655517578, + "learning_rate": 1.2542183341934873e-05, + "loss": -0.0027, + "num_tokens": 19200115.0, + "reward": 0.667292058467865, + "reward_std": 0.18406903743743896, + "rewards/cross_solution_unittest_reward/mean": 0.3336460068821907, + "rewards/cross_solution_unittest_reward/std": 0.25224795937538147, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 47 @@ -1379,24 +1379,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 840.0, - "completions/max_terminated_length": 840.0, - "completions/mean_length": 501.1015625, - "completions/mean_terminated_length": 501.1015625, - "completions/min_length": 243.0, - "completions/min_terminated_length": 243.0, - "entropy": 0.0812225341796875, + "completions/max_length": 866.0, + "completions/max_terminated_length": 866.0, + "completions/mean_length": 480.40625, + "completions/mean_terminated_length": 480.40625, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.1033172607421875, "epoch": 0.43798118049615054, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03334600478410721, - "kl": 0.003113985061645508, - "learning_rate": 1.4154150130018867e-05, - "loss": 0.0134, - "num_tokens": 19762247.0, - "reward": 0.6029683351516724, - "reward_std": 0.07560944557189941, - "rewards/cross_solution_unittest/mean": 0.6029682755470276, - "rewards/cross_solution_unittest/std": 0.112706758081913, + "grad_norm": 0.0333259254693985, + "kl": 0.003824949264526367, + "learning_rate": 1.2264967674257647e-05, + "loss": -0.0009, + "num_tokens": 19586559.0, + "reward": 0.8440192937850952, + "reward_std": 0.23113533854484558, + "rewards/cross_solution_unittest_reward/mean": 0.4220096617937088, + "rewards/cross_solution_unittest_reward/std": 0.2280796319246292, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 48 @@ -1407,25 +1407,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.953125, + "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1296.0, - "completions/mean_length": 534.4896240234375, - "completions/mean_terminated_length": 510.4776916503906, - "completions/min_length": 276.0, - "completions/min_terminated_length": 276.0, - "entropy": 0.0760498046875, + "completions/max_terminated_length": 998.0, + "completions/mean_length": 486.1458435058594, + "completions/mean_terminated_length": 478.05743408203125, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.09979248046875, "epoch": 0.4471057884231537, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03139442205429077, - "kl": 0.0026619434356689453, - "learning_rate": 1.3863451256931286e-05, - "loss": -0.0061, - "num_tokens": 20189959.0, - "reward": 0.6022586822509766, - "reward_std": 0.06519456952810287, - "rewards/cross_solution_unittest/mean": 0.6022586226463318, - "rewards/cross_solution_unittest/std": 0.11483202874660492, + "grad_norm": 0.03385407105088234, + "kl": 0.003530740737915039, + "learning_rate": 1.1985904666457455e-05, + "loss": 0.0061, + "num_tokens": 19995087.0, + "reward": 0.6481320261955261, + "reward_std": 0.26141294836997986, + "rewards/cross_solution_unittest_reward/mean": 0.3240659758448601, + "rewards/cross_solution_unittest_reward/std": 0.25914470851421356, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 49 @@ -1436,25 +1436,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1825.0, - "completions/mean_length": 540.4635620117188, - "completions/mean_terminated_length": 532.5169677734375, - "completions/min_length": 243.0, - "completions/min_terminated_length": 243.0, - "entropy": 0.0771331787109375, + "completions/clipped_ratio": -5.0, + "completions/max_length": 966.0, + "completions/max_terminated_length": 966.0, + "completions/mean_length": 514.703125, + "completions/mean_terminated_length": 514.703125, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.0972137451171875, "epoch": 0.45623039635015683, "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.03034847602248192, - "kl": 0.0031042098999023438, - "learning_rate": 1.356886221591872e-05, - "loss": 0.0005, - "num_tokens": 20614273.0, - "reward": 0.6334234476089478, - "reward_std": 0.05841737240552902, - "rewards/cross_solution_unittest/mean": 0.6334234476089478, - "rewards/cross_solution_unittest/std": 0.08473079651594162, + "grad_norm": 0.03325730189681053, + "kl": 0.004256248474121094, + "learning_rate": 1.170522192632624e-05, + "loss": -0.0051, + "num_tokens": 20408813.0, + "reward": 0.8064446449279785, + "reward_std": 0.22871556878089905, + "rewards/cross_solution_unittest_reward/mean": 0.40322232991456985, + "rewards/cross_solution_unittest_reward/std": 0.28535734862089157, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 50 @@ -1465,25 +1465,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 2980.0, - "completions/max_terminated_length": 2980.0, - "completions/mean_length": 531.890625, - "completions/mean_terminated_length": 531.890625, - "completions/min_length": 178.0, - "completions/min_terminated_length": 178.0, - "entropy": 0.07479095458984375, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 834.0, + "completions/mean_length": 510.9453125, + "completions/mean_terminated_length": 502.92169189453125, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "entropy": 0.0950927734375, "epoch": 0.46535500427716, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03556211292743683, - "kl": 0.0032439231872558594, - "learning_rate": 1.3270679633174219e-05, - "loss": 0.0069, - "num_tokens": 21040351.0, - "reward": 0.6393711566925049, - "reward_std": 0.07185273617506027, - "rewards/cross_solution_unittest/mean": 0.6393711566925049, - "rewards/cross_solution_unittest/std": 0.11849500238895416, + "grad_norm": 0.03309718891978264, + "kl": 0.004229068756103516, + "learning_rate": 1.1423148382732854e-05, + "loss": 0.0142, + "num_tokens": 20829704.0, + "reward": 0.9345626831054688, + "reward_std": 0.21369287371635437, + "rewards/cross_solution_unittest_reward/mean": 0.46728135645389557, + "rewards/cross_solution_unittest_reward/std": 0.2026936635375023, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 51 @@ -1494,25 +1494,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1223.0, - "completions/mean_length": 527.3541870117188, - "completions/mean_terminated_length": 511.3507995605469, - "completions/min_length": 201.0, - "completions/min_terminated_length": 201.0, - "entropy": 0.0787811279296875, + "completions/clipped_ratio": -5.0, + "completions/max_length": 760.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 485.2786560058594, + "completions/mean_terminated_length": 485.2786560058594, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.0979461669921875, "epoch": 0.4744796122041631, "frac_reward_zero_std": 0.0, - "grad_norm": 0.041180867701768875, - "kl": 0.0035219192504882812, - "learning_rate": 1.296920375328275e-05, - "loss": -0.0046, - "num_tokens": 21449467.0, - "reward": 0.6477403044700623, - "reward_std": 0.06604725122451782, - "rewards/cross_solution_unittest/mean": 0.6477403044700623, - "rewards/cross_solution_unittest/std": 0.12666402757167816, + "grad_norm": 0.4690159559249878, + "kl": 0.013956546783447266, + "learning_rate": 1.1139914098905406e-05, + "loss": 0.0024, + "num_tokens": 21222691.0, + "reward": 0.9599242210388184, + "reward_std": 0.23365139961242676, + "rewards/cross_solution_unittest_reward/mean": 0.479962095618248, + "rewards/cross_solution_unittest_reward/std": 0.24594762176275253, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 52 @@ -1523,25 +1523,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 943.0, - "completions/mean_length": 505.08074951171875, - "completions/mean_terminated_length": 488.9607238769531, - "completions/min_length": 235.0, - "completions/min_terminated_length": 235.0, - "entropy": 0.079986572265625, + "completions/clipped_ratio": -5.0, + "completions/max_length": 891.0, + "completions/max_terminated_length": 891.0, + "completions/mean_length": 477.48699951171875, + "completions/mean_terminated_length": 477.48699951171875, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.095062255859375, "epoch": 0.4836042201311662, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03628738969564438, - "kl": 0.004321575164794922, - "learning_rate": 1.266473813690035e-05, - "loss": 0.0045, - "num_tokens": 21858106.0, - "reward": 0.5700105428695679, - "reward_std": 0.06474246829748154, - "rewards/cross_solution_unittest/mean": 0.5700104832649231, - "rewards/cross_solution_unittest/std": 0.12813034653663635, + "grad_norm": 0.03301261365413666, + "kl": 0.0033338069915771484, + "learning_rate": 1.08557500847884e-05, + "loss": 0.0043, + "num_tokens": 21619614.0, + "reward": 0.7612521052360535, + "reward_std": 0.18586371839046478, + "rewards/cross_solution_unittest_reward/mean": 0.38062604516744614, + "rewards/cross_solution_unittest_reward/std": 0.22070425003767014, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 53 @@ -1552,25 +1552,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 975.0, - "completions/max_terminated_length": 975.0, - "completions/mean_length": 529.0443115234375, - "completions/mean_terminated_length": 529.0443115234375, - "completions/min_length": 253.0, - "completions/min_terminated_length": 253.0, - "entropy": 0.0794677734375, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 992.0, + "completions/mean_length": 529.8698120117188, + "completions/mean_terminated_length": 521.8955688476562, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.098876953125, "epoch": 0.49272882805816937, "frac_reward_zero_std": 0.0, - "grad_norm": 0.031926512718200684, - "kl": 0.003833293914794922, - "learning_rate": 1.2357589355094275e-05, - "loss": 0.0121, - "num_tokens": 22318391.0, - "reward": 0.6334779262542725, - "reward_std": 0.0830218717455864, - "rewards/cross_solution_unittest/mean": 0.6334779262542725, - "rewards/cross_solution_unittest/std": 0.1171019971370697, + "grad_norm": 0.037023600190877914, + "kl": 0.005710601806640625, + "learning_rate": 1.0570888108627682e-05, + "loss": -0.0049, + "num_tokens": 22079444.0, + "reward": 0.7765120267868042, + "reward_std": 0.25535768270492554, + "rewards/cross_solution_unittest_reward/mean": 0.3882560133934021, + "rewards/cross_solution_unittest_reward/std": 0.21029234677553177, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 54 @@ -1582,24 +1582,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1062.0, - "completions/max_terminated_length": 1062.0, - "completions/mean_length": 504.7161560058594, - "completions/mean_terminated_length": 504.7161560058594, - "completions/min_length": 267.0, - "completions/min_terminated_length": 267.0, - "entropy": 0.07977294921875, + "completions/max_length": 968.0, + "completions/max_terminated_length": 968.0, + "completions/mean_length": 480.6302185058594, + "completions/mean_terminated_length": 480.6302185058594, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.1016998291015625, "epoch": 0.5018534359851725, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03277221694588661, - "kl": 0.0038743019104003906, - "learning_rate": 1.2048066680651908e-05, - "loss": -0.005, - "num_tokens": 22699346.0, - "reward": 0.5762088298797607, - "reward_std": 0.0954534038901329, - "rewards/cross_solution_unittest/mean": 0.5762088298797607, - "rewards/cross_solution_unittest/std": 0.14616453647613525, + "grad_norm": 0.03228428214788437, + "kl": 0.003750324249267578, + "learning_rate": 1.0285560507936962e-05, + "loss": -0.0011, + "num_tokens": 22452846.0, + "reward": 0.7638306617736816, + "reward_std": 0.2091209590435028, + "rewards/cross_solution_unittest_reward/mean": 0.38191530853509903, + "rewards/cross_solution_unittest_reward/std": 0.21358844637870789, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 55 @@ -1610,25 +1610,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 2795.0, - "completions/mean_length": 505.390625, - "completions/mean_terminated_length": 497.35247802734375, - "completions/min_length": 256.0, - "completions/min_terminated_length": 256.0, - "entropy": 0.07952117919921875, + "completions/clipped_ratio": -5.0, + "completions/max_length": 886.0, + "completions/max_terminated_length": 886.0, + "completions/mean_length": 487.8567810058594, + "completions/mean_terminated_length": 487.8567810058594, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.1000823974609375, "epoch": 0.5109780439121756, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03929758444428444, - "kl": 0.005110740661621094, - "learning_rate": 1.1736481776669307e-05, - "loss": 0.003, - "num_tokens": 23081156.0, - "reward": 0.5718110799789429, - "reward_std": 0.08531081676483154, - "rewards/cross_solution_unittest/mean": 0.5718110203742981, - "rewards/cross_solution_unittest/std": 0.12530149519443512, + "grad_norm": 0.032228607684373856, + "kl": 0.0042340755462646484, + "learning_rate": 1e-05, + "loss": 0.0103, + "num_tokens": 22825151.0, + "reward": 0.826278567314148, + "reward_std": 0.19069427251815796, + "rewards/cross_solution_unittest_reward/mean": 0.4131392240524292, + "rewards/cross_solution_unittest_reward/std": 0.21790383756160736, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 56 @@ -1639,25 +1639,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1219.0, - "completions/mean_length": 516.2057495117188, - "completions/mean_terminated_length": 508.1958312988281, - "completions/min_length": 280.0, - "completions/min_terminated_length": 280.0, - "entropy": 0.0814666748046875, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1560.0, + "completions/max_terminated_length": 1560.0, + "completions/mean_length": 494.5520935058594, + "completions/mean_terminated_length": 494.5520935058594, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.099639892578125, "epoch": 0.5201026518391788, "frac_reward_zero_std": 0.0, - "grad_norm": 0.0367487296462059, - "kl": 0.004551410675048828, - "learning_rate": 1.1423148382732854e-05, - "loss": 0.0163, - "num_tokens": 23529571.0, - "reward": 0.6225696802139282, - "reward_std": 0.060477182269096375, - "rewards/cross_solution_unittest/mean": 0.6225696802139282, - "rewards/cross_solution_unittest/std": 0.1196565330028534, + "grad_norm": 1.5714826583862305, + "kl": 0.06778192520141602, + "learning_rate": 9.71443949206304e-06, + "loss": 0.0054, + "num_tokens": 23267683.0, + "reward": 0.8670098185539246, + "reward_std": 0.18901364505290985, + "rewards/cross_solution_unittest_reward/mean": 0.4335048869252205, + "rewards/cross_solution_unittest_reward/std": 0.2112448662519455, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 57 @@ -1670,23 +1670,23 @@ "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.953125, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1001.0, - "completions/mean_length": 507.19793701171875, - "completions/mean_terminated_length": 482.97113037109375, - "completions/min_length": 219.0, - "completions/min_terminated_length": 219.0, - "entropy": 0.088714599609375, + "completions/max_terminated_length": 966.0, + "completions/mean_length": 494.9739685058594, + "completions/mean_terminated_length": 470.6509094238281, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.1165924072265625, "epoch": 0.5292272597661819, "frac_reward_zero_std": 0.0, - "grad_norm": 0.036932192742824554, - "kl": 0.005054950714111328, - "learning_rate": 1.1108381999010111e-05, - "loss": -0.0237, - "num_tokens": 23952099.0, - "reward": 0.61222243309021, - "reward_std": 0.07403381913900375, - "rewards/cross_solution_unittest/mean": 0.61222243309021, - "rewards/cross_solution_unittest/std": 0.12056523561477661, + "grad_norm": 0.03703828528523445, + "kl": 0.004994392395019531, + "learning_rate": 9.42911189137232e-06, + "loss": 0.0108, + "num_tokens": 23682969.0, + "reward": 0.8357396125793457, + "reward_std": 0.1958376169204712, + "rewards/cross_solution_unittest_reward/mean": 0.41786976903676987, + "rewards/cross_solution_unittest_reward/std": 0.28136271238327026, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 58 @@ -1697,25 +1697,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.9375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 3118.0, - "completions/mean_length": 512.6849365234375, - "completions/mean_terminated_length": 480.35528564453125, - "completions/min_length": 241.0, - "completions/min_terminated_length": 241.0, - "entropy": 0.0914764404296875, + "completions/clipped_ratio": -5.0, + "completions/max_length": 906.0, + "completions/max_terminated_length": 906.0, + "completions/mean_length": 464.0755310058594, + "completions/mean_terminated_length": 464.0755310058594, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.1112823486328125, "epoch": 0.5383518676931851, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03964272886514664, - "kl": 0.006877422332763672, - "learning_rate": 1.0792499568567885e-05, - "loss": 0.0078, - "num_tokens": 24370834.0, - "reward": 0.6122474074363708, - "reward_std": 0.062438443303108215, - "rewards/cross_solution_unittest/mean": 0.6122473478317261, - "rewards/cross_solution_unittest/std": 0.11854556947946548, + "grad_norm": 0.03699680417776108, + "kl": 0.004660606384277344, + "learning_rate": 9.144249915211605e-06, + "loss": -0.0151, + "num_tokens": 24082606.0, + "reward": 0.8085569739341736, + "reward_std": 0.1914159059524536, + "rewards/cross_solution_unittest_reward/mean": 0.4042784869670868, + "rewards/cross_solution_unittest_reward/std": 0.2565099969506264, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 59 @@ -1727,24 +1727,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1433.0, - "completions/max_terminated_length": 1433.0, - "completions/mean_length": 510.22918701171875, - "completions/mean_terminated_length": 510.22918701171875, - "completions/min_length": 283.0, - "completions/min_terminated_length": 283.0, - "entropy": 0.0788726806640625, + "completions/max_length": 1127.0, + "completions/max_terminated_length": 1127.0, + "completions/mean_length": 488.7890625, + "completions/mean_terminated_length": 488.7890625, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.093719482421875, "epoch": 0.5474764756201882, "frac_reward_zero_std": 0.0, - "grad_norm": 0.037243351340293884, - "kl": 0.005549430847167969, - "learning_rate": 1.0475819158237426e-05, - "loss": 0.0003, - "num_tokens": 24808546.0, - "reward": 0.6103326082229614, - "reward_std": 0.0680248886346817, - "rewards/cross_solution_unittest/mean": 0.6103325486183167, - "rewards/cross_solution_unittest/std": 0.11332948505878448, + "grad_norm": 0.030238306149840355, + "kl": 0.004515171051025391, + "learning_rate": 8.860085901094595e-06, + "loss": 0.003, + "num_tokens": 24514333.0, + "reward": 0.8814453482627869, + "reward_std": 0.23693448305130005, + "rewards/cross_solution_unittest_reward/mean": 0.44072266668081284, + "rewards/cross_solution_unittest_reward/std": 0.22619559615850449, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 60 @@ -1756,24 +1756,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1041.0, - "completions/max_terminated_length": 1041.0, - "completions/mean_length": 544.8724365234375, - "completions/mean_terminated_length": 544.8724365234375, - "completions/min_length": 274.0, - "completions/min_terminated_length": 274.0, - "entropy": 0.07517242431640625, + "completions/max_length": 984.0, + "completions/max_terminated_length": 984.0, + "completions/mean_length": 526.2005615234375, + "completions/mean_terminated_length": 526.2005615234375, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.0959625244140625, "epoch": 0.5566010835471913, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03436488285660744, - "kl": 0.004729270935058594, - "learning_rate": 1.015865963834808e-05, - "loss": 0.0028, - "num_tokens": 25265381.0, - "reward": 0.5960478782653809, - "reward_std": 0.07282952964305878, - "rewards/cross_solution_unittest/mean": 0.5960478782653809, - "rewards/cross_solution_unittest/std": 0.12416920810937881, + "grad_norm": 0.03212675452232361, + "kl": 0.0042629241943359375, + "learning_rate": 8.576851617267151e-06, + "loss": -0.0098, + "num_tokens": 24961698.0, + "reward": 0.8686231374740601, + "reward_std": 0.19413742423057556, + "rewards/cross_solution_unittest_reward/mean": 0.43431156873703003, + "rewards/cross_solution_unittest_reward/std": 0.26492878049612045, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 61 @@ -1785,24 +1785,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1118.0, - "completions/max_terminated_length": 1118.0, - "completions/mean_length": 525.9349365234375, - "completions/mean_terminated_length": 525.9349365234375, - "completions/min_length": 271.0, - "completions/min_terminated_length": 271.0, - "entropy": 0.0847625732421875, + "completions/max_length": 919.0, + "completions/max_terminated_length": 919.0, + "completions/mean_length": 520.0104370117188, + "completions/mean_terminated_length": 520.0104370117188, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "entropy": 0.1033935546875, "epoch": 0.5657256914741945, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03560968488454819, - "kl": 0.005660057067871094, - "learning_rate": 9.841340361651921e-06, - "loss": 0.0144, - "num_tokens": 25690764.0, - "reward": 0.6058902144432068, - "reward_std": 0.0673966109752655, - "rewards/cross_solution_unittest/mean": 0.6058902144432068, - "rewards/cross_solution_unittest/std": 0.0988718718290329, + "grad_norm": 0.0341770313680172, + "kl": 0.004704952239990234, + "learning_rate": 8.294778073673762e-06, + "loss": 0.0204, + "num_tokens": 25386422.0, + "reward": 0.747821033000946, + "reward_std": 0.21126197278499603, + "rewards/cross_solution_unittest_reward/mean": 0.37391049042344093, + "rewards/cross_solution_unittest_reward/std": 0.19313736259937286, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 62 @@ -1813,25 +1813,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.953125, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 972.0, - "completions/mean_length": 502.2083435058594, - "completions/mean_terminated_length": 477.9422607421875, - "completions/min_length": 223.0, - "completions/min_terminated_length": 223.0, - "entropy": 0.08184814453125, + "completions/clipped_ratio": -5.0, + "completions/max_length": 802.0, + "completions/max_terminated_length": 802.0, + "completions/mean_length": 457.11199951171875, + "completions/mean_terminated_length": 457.11199951171875, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.1014862060546875, "epoch": 0.5748502994011976, "frac_reward_zero_std": 0.0, - "grad_norm": 0.037654757499694824, - "kl": 0.0056934356689453125, - "learning_rate": 9.524180841762577e-06, - "loss": 0.0073, - "num_tokens": 26075548.0, - "reward": 0.6195604801177979, - "reward_std": 0.09131857007741928, - "rewards/cross_solution_unittest/mean": 0.6195604205131531, - "rewards/cross_solution_unittest/std": 0.12209580838680267, + "grad_norm": 0.03489193320274353, + "kl": 0.004260540008544922, + "learning_rate": 8.014095333542548e-06, + "loss": -0.0047, + "num_tokens": 25751057.0, + "reward": 0.7548073530197144, + "reward_std": 0.20320609211921692, + "rewards/cross_solution_unittest_reward/mean": 0.3774036541581154, + "rewards/cross_solution_unittest_reward/std": 0.21862108260393143, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 63 @@ -1842,25 +1842,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1050.0, - "completions/max_terminated_length": 1050.0, - "completions/mean_length": 504.7395935058594, - "completions/mean_terminated_length": 504.7395935058594, - "completions/min_length": 280.0, - "completions/min_terminated_length": 280.0, - "entropy": 0.077972412109375, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 842.0, + "completions/mean_length": 497.3567810058594, + "completions/mean_terminated_length": 489.29766845703125, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.09716796875, "epoch": 0.5839749073282008, "frac_reward_zero_std": 0.0, - "grad_norm": 0.04175296798348427, - "kl": 0.005429267883300781, - "learning_rate": 9.207500431432115e-06, - "loss": -0.0079, - "num_tokens": 26471072.0, - "reward": 0.593117356300354, - "reward_std": 0.08295892179012299, - "rewards/cross_solution_unittest/mean": 0.5931172966957092, - "rewards/cross_solution_unittest/std": 0.1298408806324005, + "grad_norm": 0.04765147343277931, + "kl": 0.00534820556640625, + "learning_rate": 7.735032325742355e-06, + "loss": -0.0042, + "num_tokens": 26145418.0, + "reward": 1.0832295417785645, + "reward_std": 0.20443449914455414, + "rewards/cross_solution_unittest_reward/mean": 0.5416147857904434, + "rewards/cross_solution_unittest_reward/std": 0.1956881359219551, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 64 @@ -1872,24 +1872,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 849.0, - "completions/max_terminated_length": 849.0, - "completions/mean_length": 464.1145935058594, - "completions/mean_terminated_length": 464.1145935058594, - "completions/min_length": 184.0, - "completions/min_terminated_length": 184.0, - "entropy": 0.089385986328125, + "completions/max_length": 1014.0, + "completions/max_terminated_length": 1014.0, + "completions/mean_length": 469.390625, + "completions/mean_terminated_length": 469.390625, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "entropy": 0.110504150390625, "epoch": 0.5930995152552039, "frac_reward_zero_std": 0.0, - "grad_norm": 0.039406850934028625, - "kl": 0.006228446960449219, - "learning_rate": 8.89161800098989e-06, - "loss": -0.0001, - "num_tokens": 26854340.0, - "reward": 0.6415243148803711, - "reward_std": 0.0817657858133316, - "rewards/cross_solution_unittest/mean": 0.6415242552757263, - "rewards/cross_solution_unittest/std": 0.13109222054481506, + "grad_norm": 0.033948320895433426, + "kl": 0.004923343658447266, + "learning_rate": 7.4578166580651335e-06, + "loss": -0.0063, + "num_tokens": 26531240.0, + "reward": 0.9129461050033569, + "reward_std": 0.2018149346113205, + "rewards/cross_solution_unittest_reward/mean": 0.4564730077981949, + "rewards/cross_solution_unittest_reward/std": 0.2923680394887924, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 65 @@ -1900,25 +1900,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, + "completions/clipped_ratio": -4.953125, "completions/max_length": 3584.0, - "completions/max_terminated_length": 2036.0, - "completions/mean_length": 537.296875, - "completions/mean_terminated_length": 521.3455810546875, - "completions/min_length": 241.0, - "completions/min_terminated_length": 241.0, - "entropy": 0.082550048828125, + "completions/max_terminated_length": 1764.0, + "completions/mean_length": 536.3333740234375, + "completions/mean_terminated_length": 512.3359375, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.0997772216796875, "epoch": 0.602224123182207, "frac_reward_zero_std": 0.0, - "grad_norm": 0.034542761743068695, - "kl": 0.005867958068847656, - "learning_rate": 8.576851617267151e-06, - "loss": -0.0014, - "num_tokens": 27332642.0, - "reward": 0.6036405563354492, - "reward_std": 0.0659734234213829, - "rewards/cross_solution_unittest/mean": 0.6036404967308044, - "rewards/cross_solution_unittest/std": 0.10401112586259842, + "grad_norm": 0.030758032575249672, + "kl": 0.004119873046875, + "learning_rate": 7.182674431585703e-06, + "loss": 0.0112, + "num_tokens": 27009184.0, + "reward": 0.9593736529350281, + "reward_std": 0.22100883722305298, + "rewards/cross_solution_unittest_reward/mean": 0.47968684136867523, + "rewards/cross_solution_unittest_reward/std": 0.2801002562046051, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 66 @@ -1929,25 +1929,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 956.0, - "completions/mean_length": 511.71875, - "completions/mean_terminated_length": 495.6335144042969, - "completions/min_length": 235.0, - "completions/min_terminated_length": 235.0, - "entropy": 0.0772247314453125, + "completions/clipped_ratio": -5.0, + "completions/max_length": 792.0, + "completions/max_terminated_length": 792.0, + "completions/mean_length": 486.00262451171875, + "completions/mean_terminated_length": 486.00262451171875, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.0943145751953125, "epoch": 0.6113487311092102, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03523527458310127, - "kl": 0.0067005157470703125, - "learning_rate": 8.263518223330698e-06, - "loss": 0.0031, - "num_tokens": 27795114.0, - "reward": 0.6518998146057129, - "reward_std": 0.06018558144569397, - "rewards/cross_solution_unittest/mean": 0.6518998146057129, - "rewards/cross_solution_unittest/std": 0.0940474197268486, + "grad_norm": 0.03071185201406479, + "kl": 0.004786491394042969, + "learning_rate": 6.909830056250527e-06, + "loss": 0.0004, + "num_tokens": 27460937.0, + "reward": 1.0903098583221436, + "reward_std": 0.25236761569976807, + "rewards/cross_solution_unittest_reward/mean": 0.5451549142599106, + "rewards/cross_solution_unittest_reward/std": 0.24181817471981049, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 67 @@ -1958,25 +1958,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1068.0, - "completions/mean_length": 534.8724365234375, - "completions/mean_terminated_length": 526.9112548828125, - "completions/min_length": 280.0, - "completions/min_terminated_length": 280.0, - "entropy": 0.0787353515625, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1091.0, + "completions/max_terminated_length": 1091.0, + "completions/mean_length": 514.515625, + "completions/mean_terminated_length": 514.515625, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.0963897705078125, "epoch": 0.6204733390362133, "frac_reward_zero_std": 0.0, - "grad_norm": 0.031150035560131073, - "kl": 0.005561351776123047, - "learning_rate": 7.951933319348095e-06, - "loss": -0.0163, - "num_tokens": 28203505.0, - "reward": 0.6172535419464111, - "reward_std": 0.07265746593475342, - "rewards/cross_solution_unittest/mean": 0.6172534823417664, - "rewards/cross_solution_unittest/std": 0.10251819342374802, + "grad_norm": 0.030294137075543404, + "kl": 0.0035681724548339844, + "learning_rate": 6.639506067845698e-06, + "loss": 0.0073, + "num_tokens": 27860791.0, + "reward": 0.9012066721916199, + "reward_std": 0.18023371696472168, + "rewards/cross_solution_unittest_reward/mean": 0.45060331374406815, + "rewards/cross_solution_unittest_reward/std": 0.19457216560840607, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 68 @@ -1988,24 +1988,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 2633.0, - "completions/max_terminated_length": 2633.0, - "completions/mean_length": 504.0520935058594, - "completions/mean_terminated_length": 504.0520935058594, - "completions/min_length": 195.0, - "completions/min_terminated_length": 195.0, - "entropy": 0.0829925537109375, + "completions/max_length": 2792.0, + "completions/max_terminated_length": 2792.0, + "completions/mean_length": 488.8046875, + "completions/mean_terminated_length": 488.8046875, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.100738525390625, "epoch": 0.6295979469632165, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03765825554728508, - "kl": 0.007781982421875, - "learning_rate": 7.642410644905726e-06, - "loss": -0.0102, - "num_tokens": 28643677.0, - "reward": 0.5815111398696899, - "reward_std": 0.07605060935020447, - "rewards/cross_solution_unittest/mean": 0.5815110802650452, - "rewards/cross_solution_unittest/std": 0.1344773769378662, + "grad_norm": 0.029943598434329033, + "kl": 0.004576683044433594, + "learning_rate": 6.3719229464935915e-06, + "loss": 0.0022, + "num_tokens": 28294524.0, + "reward": 0.9089535474777222, + "reward_std": 0.21831755340099335, + "rewards/cross_solution_unittest_reward/mean": 0.4544767662882805, + "rewards/cross_solution_unittest_reward/std": 0.20777993649244308, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 69 @@ -2017,24 +2017,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1305.0, - "completions/max_terminated_length": 1305.0, - "completions/mean_length": 521.3515625, - "completions/mean_terminated_length": 521.3515625, - "completions/min_length": 235.0, - "completions/min_terminated_length": 235.0, - "entropy": 0.076080322265625, + "completions/max_length": 1007.0, + "completions/max_terminated_length": 1007.0, + "completions/mean_length": 517.0234375, + "completions/mean_terminated_length": 517.0234375, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.09405517578125, "epoch": 0.6387225548902196, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03361423686146736, - "kl": 0.005164623260498047, - "learning_rate": 7.335261863099652e-06, - "loss": -0.0013, - "num_tokens": 29087396.0, - "reward": 0.5668891668319702, - "reward_std": 0.0703488141298294, - "rewards/cross_solution_unittest/mean": 0.5668891072273254, - "rewards/cross_solution_unittest/std": 0.1609523743391037, + "grad_norm": 0.032788679003715515, + "kl": 0.00408935546875, + "learning_rate": 6.107298936826086e-06, + "loss": 0.003, + "num_tokens": 28738997.0, + "reward": 0.9689019918441772, + "reward_std": 0.2762044668197632, + "rewards/cross_solution_unittest_reward/mean": 0.4844510108232498, + "rewards/cross_solution_unittest_reward/std": 0.21708913147449493, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 70 @@ -2045,25 +2045,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.9375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1279.0, - "completions/mean_length": 543.7734375, - "completions/mean_terminated_length": 511.7710876464844, - "completions/min_length": 283.0, - "completions/min_terminated_length": 283.0, - "entropy": 0.08349609375, + "completions/clipped_ratio": -5.0, + "completions/max_length": 898.0, + "completions/max_terminated_length": 898.0, + "completions/mean_length": 492.09637451171875, + "completions/mean_terminated_length": 492.09637451171875, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.10223388671875, "epoch": 0.6478471628172227, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03806523233652115, - "kl": 0.006535530090332031, - "learning_rate": 7.0307962467172555e-06, - "loss": 0.009, - "num_tokens": 29509857.0, - "reward": 0.5793834924697876, - "reward_std": 0.10175223648548126, - "rewards/cross_solution_unittest/mean": 0.5793834328651428, - "rewards/cross_solution_unittest/std": 0.13090623915195465, + "grad_norm": 0.03405945748090744, + "kl": 0.004323005676269531, + "learning_rate": 5.845849869981137e-06, + "loss": -0.0087, + "num_tokens": 29140394.0, + "reward": 0.6452049016952515, + "reward_std": 0.18944227695465088, + "rewards/cross_solution_unittest_reward/mean": 0.32260240986943245, + "rewards/cross_solution_unittest_reward/std": 0.19827812910079956, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 71 @@ -2074,25 +2074,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1215.0, - "completions/mean_length": 522.6796875, - "completions/mean_terminated_length": 514.6867065429688, - "completions/min_length": 247.0, - "completions/min_terminated_length": 247.0, - "entropy": 0.08115386962890625, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1050.0, + "completions/max_terminated_length": 1050.0, + "completions/mean_length": 511.7578125, + "completions/mean_terminated_length": 511.7578125, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.0999755859375, "epoch": 0.6569717707442259, "frac_reward_zero_std": 0.0, - "grad_norm": 0.038980960845947266, - "kl": 0.007760047912597656, - "learning_rate": 6.729320366825785e-06, - "loss": 0.0078, - "num_tokens": 29920282.0, - "reward": 0.623957097530365, - "reward_std": 0.05566735565662384, - "rewards/cross_solution_unittest/mean": 0.6239569783210754, - "rewards/cross_solution_unittest/std": 0.08572336286306381, + "grad_norm": 0.04706510528922081, + "kl": 0.005205631256103516, + "learning_rate": 5.587788987567785e-06, + "loss": 0.0065, + "num_tokens": 29547821.0, + "reward": 0.9646930694580078, + "reward_std": 0.1975850760936737, + "rewards/cross_solution_unittest_reward/mean": 0.4823464900255203, + "rewards/cross_solution_unittest_reward/std": 0.26742689311504364, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 72 @@ -2103,25 +2103,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1231.0, - "completions/mean_length": 501.3255310058594, - "completions/mean_terminated_length": 485.1858825683594, - "completions/min_length": 202.0, - "completions/min_terminated_length": 202.0, - "entropy": 0.0852813720703125, + "completions/clipped_ratio": -5.0, + "completions/max_length": 960.0, + "completions/max_terminated_length": 960.0, + "completions/mean_length": 469.0859375, + "completions/mean_terminated_length": 469.0859375, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "entropy": 0.103759765625, "epoch": 0.666096378671229, "frac_reward_zero_std": 0.0, - "grad_norm": 0.040267642587423325, - "kl": 0.007144451141357422, - "learning_rate": 6.431137784081283e-06, - "loss": -0.0006, - "num_tokens": 30330855.0, - "reward": 0.6102144122123718, - "reward_std": 0.06656725704669952, - "rewards/cross_solution_unittest/mean": 0.6102144122123718, - "rewards/cross_solution_unittest/std": 0.10697145015001297, + "grad_norm": 0.062288798391819, + "kl": 0.007381916046142578, + "learning_rate": 5.333326767743263e-06, + "loss": -0.0033, + "num_tokens": 29945526.0, + "reward": 1.0015891790390015, + "reward_std": 0.24343864619731903, + "rewards/cross_solution_unittest_reward/mean": 0.5007946044206619, + "rewards/cross_solution_unittest_reward/std": 0.24663101881742477, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 73 @@ -2133,24 +2133,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1347.0, - "completions/max_terminated_length": 1347.0, - "completions/mean_length": 525.5677490234375, - "completions/mean_terminated_length": 525.5677490234375, - "completions/min_length": 206.0, - "completions/min_terminated_length": 206.0, - "entropy": 0.0807037353515625, + "completions/max_length": 970.0, + "completions/max_terminated_length": 970.0, + "completions/mean_length": 514.5989990234375, + "completions/mean_terminated_length": 514.5989990234375, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "entropy": 0.0988922119140625, "epoch": 0.6752209865982322, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03388255834579468, - "kl": 0.007240772247314453, - "learning_rate": 6.136548743068713e-06, - "loss": 0.0082, - "num_tokens": 30740081.0, - "reward": 0.6364445686340332, - "reward_std": 0.06512817740440369, - "rewards/cross_solution_unittest/mean": 0.6364445686340332, - "rewards/cross_solution_unittest/std": 0.13945284485816956, + "grad_norm": 0.034228503704071045, + "kl": 0.004460811614990234, + "learning_rate": 5.082670753543961e-06, + "loss": 0.0019, + "num_tokens": 30349572.0, + "reward": 0.8880095481872559, + "reward_std": 0.2291492521762848, + "rewards/cross_solution_unittest_reward/mean": 0.44400477409362793, + "rewards/cross_solution_unittest_reward/std": 0.24787399172782898, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 74 @@ -2161,25 +2161,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1336.0, - "completions/max_terminated_length": 1336.0, - "completions/mean_length": 518.7994995117188, - "completions/mean_terminated_length": 518.7994995117188, - "completions/min_length": 272.0, - "completions/min_terminated_length": 272.0, - "entropy": 0.091766357421875, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1092.0, + "completions/mean_length": 510.9739685058594, + "completions/mean_terminated_length": 502.9504089355469, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.106231689453125, "epoch": 0.6843455945252352, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03697756305336952, - "kl": 0.006485462188720703, - "learning_rate": 5.845849869981137e-06, - "loss": -0.0074, - "num_tokens": 31119772.0, - "reward": 0.5954554080963135, - "reward_std": 0.0874066948890686, - "rewards/cross_solution_unittest/mean": 0.5954554677009583, - "rewards/cross_solution_unittest/std": 0.10502495616674423, + "grad_norm": 0.030502676963806152, + "kl": 0.003562450408935547, + "learning_rate": 4.836025383610382e-06, + "loss": -0.0003, + "num_tokens": 30727274.0, + "reward": 0.9304172396659851, + "reward_std": 0.20447766780853271, + "rewards/cross_solution_unittest_reward/mean": 0.46520860493183136, + "rewards/cross_solution_unittest_reward/std": 0.23417390882968903, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 75 @@ -2190,25 +2190,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 992.0, - "completions/mean_length": 470.27606201171875, - "completions/mean_terminated_length": 462.1462097167969, - "completions/min_length": 239.0, - "completions/min_terminated_length": 239.0, - "entropy": 0.09112548828125, + "completions/clipped_ratio": -5.0, + "completions/max_length": 888.0, + "completions/max_terminated_length": 888.0, + "completions/mean_length": 464.9453125, + "completions/mean_terminated_length": 464.9453125, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.1075897216796875, "epoch": 0.6934702024522383, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03978769853711128, - "kl": 0.008324623107910156, - "learning_rate": 5.559333873942259e-06, - "loss": 0.0145, - "num_tokens": 31463730.0, - "reward": 0.6490760445594788, - "reward_std": 0.06858085095882416, - "rewards/cross_solution_unittest/mean": 0.6490760445594788, - "rewards/cross_solution_unittest/std": 0.1581326723098755, + "grad_norm": 0.03586795926094055, + "kl": 0.004437446594238281, + "learning_rate": 4.593591825444028e-06, + "loss": 0.0051, + "num_tokens": 31071701.0, + "reward": 0.7625653147697449, + "reward_std": 0.1792157143354416, + "rewards/cross_solution_unittest_reward/mean": 0.38128262013196945, + "rewards/cross_solution_unittest_reward/std": 0.22236895561218262, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 76 @@ -2219,25 +2219,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, + "completions/clipped_ratio": -4.96875, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1465.0, - "completions/mean_length": 508.54949951171875, - "completions/mean_terminated_length": 500.51959228515625, - "completions/min_length": 242.0, - "completions/min_terminated_length": 242.0, - "entropy": 0.0853118896484375, + "completions/max_terminated_length": 890.0, + "completions/mean_length": 497.3125, + "completions/mean_terminated_length": 481.1518249511719, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.105224609375, "epoch": 0.7025948103792415, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03948879987001419, - "kl": 0.00698089599609375, - "learning_rate": 5.277289252273175e-06, - "loss": 0.0121, - "num_tokens": 31872545.0, - "reward": 0.5989640951156616, - "reward_std": 0.10313881933689117, - "rewards/cross_solution_unittest/mean": 0.5989640355110168, - "rewards/cross_solution_unittest/std": 0.14925776422023773, + "grad_norm": 0.03737795352935791, + "kl": 0.003971576690673828, + "learning_rate": 4.355567811332311e-06, + "loss": -0.0003, + "num_tokens": 31477149.0, + "reward": 0.9584690928459167, + "reward_std": 0.2223634421825409, + "rewards/cross_solution_unittest_reward/mean": 0.47923456132411957, + "rewards/cross_solution_unittest_reward/std": 0.20396708697080612, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 77 @@ -2248,25 +2248,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 2062.0, - "completions/mean_length": 547.5208740234375, - "completions/mean_terminated_length": 539.5927124023438, - "completions/min_length": 237.0, - "completions/min_terminated_length": 237.0, - "entropy": 0.0765838623046875, + "completions/clipped_ratio": -5.0, + "completions/max_length": 880.0, + "completions/max_terminated_length": 880.0, + "completions/mean_length": 495.7161560058594, + "completions/mean_terminated_length": 495.7161560058594, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.097930908203125, "epoch": 0.7117194183062446, "frac_reward_zero_std": 0.0, - "grad_norm": 0.04029017686843872, - "kl": 0.00771331787109375, - "learning_rate": 5.000000000000003e-06, - "loss": -0.0006, - "num_tokens": 32327125.0, - "reward": 0.6118708848953247, - "reward_std": 0.08621498942375183, - "rewards/cross_solution_unittest/mean": 0.6118709444999695, - "rewards/cross_solution_unittest/std": 0.146837517619133, + "grad_norm": 0.159642294049263, + "kl": 0.006262063980102539, + "learning_rate": 4.12214747707527e-06, + "loss": -0.0001, + "num_tokens": 31912712.0, + "reward": 0.9255527257919312, + "reward_std": 0.2571796178817749, + "rewards/cross_solution_unittest_reward/mean": 0.46277637779712677, + "rewards/cross_solution_unittest_reward/std": 0.25626206398010254, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 78 @@ -2277,25 +2277,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.875, + "completions/clipped_ratio": -4.9375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1261.0, - "completions/mean_length": 568.5625, - "completions/mean_terminated_length": 504.40423583984375, - "completions/min_length": 235.0, - "completions/min_terminated_length": 235.0, - "entropy": 0.0814666748046875, + "completions/max_terminated_length": 1027.0, + "completions/mean_length": 512.7135620117188, + "completions/mean_terminated_length": 480.38421630859375, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "entropy": 0.1019744873046875, "epoch": 0.7208440262332478, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03578219935297966, - "kl": 0.007190704345703125, - "learning_rate": 4.727745323894976e-06, - "loss": -0.0134, - "num_tokens": 32743281.0, - "reward": 0.6152719855308533, - "reward_std": 0.061369992792606354, - "rewards/cross_solution_unittest/mean": 0.6152719855308533, - "rewards/cross_solution_unittest/std": 0.12953531742095947, + "grad_norm": 0.032687410712242126, + "kl": 0.004110813140869141, + "learning_rate": 3.893521203645618e-06, + "loss": -0.0029, + "num_tokens": 32308370.0, + "reward": 0.9185789823532104, + "reward_std": 0.19305042922496796, + "rewards/cross_solution_unittest_reward/mean": 0.45928946137428284, + "rewards/cross_solution_unittest_reward/std": 0.23322047293186188, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 79 @@ -2306,25 +2306,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1041.0, - "completions/mean_length": 498.3177185058594, - "completions/mean_terminated_length": 490.2611083984375, - "completions/min_length": 226.0, - "completions/min_terminated_length": 226.0, - "entropy": 0.0795440673828125, + "completions/clipped_ratio": -5.0, + "completions/max_length": 938.0, + "completions/max_terminated_length": 938.0, + "completions/mean_length": 471.16668701171875, + "completions/mean_terminated_length": 471.16668701171875, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.0972900390625, "epoch": 0.7299686341602509, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03637859597802162, - "kl": 0.00676727294921875, - "learning_rate": 4.460799361338898e-06, - "loss": -0.0083, - "num_tokens": 33113451.0, - "reward": 0.6233752369880676, - "reward_std": 0.0716649666428566, - "rewards/cross_solution_unittest/mean": 0.6233752369880676, - "rewards/cross_solution_unittest/std": 0.14779163897037506, + "grad_norm": 0.034349896013736725, + "kl": 0.0041866302490234375, + "learning_rate": 3.6698754619112974e-06, + "loss": -0.0032, + "num_tokens": 32667882.0, + "reward": 0.8720422983169556, + "reward_std": 0.18141961097717285, + "rewards/cross_solution_unittest_reward/mean": 0.436021126806736, + "rewards/cross_solution_unittest_reward/std": 0.23245983570814133, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 80 @@ -2335,25 +2335,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 2090.0, - "completions/mean_length": 507.8724060058594, - "completions/mean_terminated_length": 499.8407287597656, - "completions/min_length": 171.0, - "completions/min_terminated_length": 171.0, - "entropy": 0.08404541015625, + "completions/clipped_ratio": -5.0, + "completions/max_length": 952.0, + "completions/max_terminated_length": 952.0, + "completions/mean_length": 484.87762451171875, + "completions/mean_terminated_length": 484.87762451171875, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "entropy": 0.099090576171875, "epoch": 0.739093242087254, "frac_reward_zero_std": 0.0, - "grad_norm": 0.034254688769578934, - "kl": 0.006606101989746094, - "learning_rate": 4.19943090428802e-06, - "loss": 0.0015, - "num_tokens": 33504634.0, - "reward": 0.5900766849517822, - "reward_std": 0.07616183161735535, - "rewards/cross_solution_unittest/mean": 0.5900766253471375, - "rewards/cross_solution_unittest/std": 0.15281453728675842, + "grad_norm": 0.03139191120862961, + "kl": 0.0039272308349609375, + "learning_rate": 3.4513926605471504e-06, + "loss": 0.0016, + "num_tokens": 33050987.0, + "reward": 0.9457516074180603, + "reward_std": 0.1805034577846527, + "rewards/cross_solution_unittest_reward/mean": 0.47287583351135254, + "rewards/cross_solution_unittest_reward/std": 0.19942139089107513, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 81 @@ -2364,25 +2364,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1011.0, - "completions/mean_length": 493.8177185058594, - "completions/mean_terminated_length": 485.7493591308594, - "completions/min_length": 226.0, - "completions/min_terminated_length": 226.0, - "entropy": 0.08441162109375, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1118.0, + "completions/max_terminated_length": 1118.0, + "completions/mean_length": 475.60418701171875, + "completions/mean_terminated_length": 475.60418701171875, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 0.1024169921875, "epoch": 0.7482178500142572, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03650706261396408, - "kl": 0.007792472839355469, - "learning_rate": 3.943903128623336e-06, - "loss": -0.0018, - "num_tokens": 33896656.0, - "reward": 0.6140320301055908, - "reward_std": 0.07920640707015991, - "rewards/cross_solution_unittest/mean": 0.614031970500946, - "rewards/cross_solution_unittest/std": 0.12956902384757996, + "grad_norm": 0.030605316162109375, + "kl": 0.004054546356201172, + "learning_rate": 3.2382509972598087e-06, + "loss": 0.0052, + "num_tokens": 33439955.0, + "reward": 0.8594505190849304, + "reward_std": 0.20357158780097961, + "rewards/cross_solution_unittest_reward/mean": 0.4297252967953682, + "rewards/cross_solution_unittest_reward/std": 0.24937064200639725, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 82 @@ -2393,25 +2393,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, + "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 866.0, - "completions/mean_length": 524.1484375, - "completions/mean_terminated_length": 508.1282653808594, - "completions/min_length": 265.0, - "completions/min_terminated_length": 265.0, - "entropy": 0.081451416015625, + "completions/max_terminated_length": 808.0, + "completions/mean_length": 483.18231201171875, + "completions/mean_terminated_length": 475.086181640625, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.101776123046875, "epoch": 0.7573424579412603, "frac_reward_zero_std": 0.0, - "grad_norm": 0.034576863050460815, - "kl": 0.007690906524658203, - "learning_rate": 3.6944733291547784e-06, - "loss": -0.0122, - "num_tokens": 34306081.0, - "reward": 0.6044012308120728, - "reward_std": 0.07356821745634079, - "rewards/cross_solution_unittest/mean": 0.6044012904167175, - "rewards/cross_solution_unittest/std": 0.12973514199256897, + "grad_norm": 0.034236159175634384, + "kl": 0.004290103912353516, + "learning_rate": 3.0306243134470668e-06, + "loss": 0.0102, + "num_tokens": 33834753.0, + "reward": 0.8233698606491089, + "reward_std": 0.25579404830932617, + "rewards/cross_solution_unittest_reward/mean": 0.41168488562107086, + "rewards/cross_solution_unittest_reward/std": 0.23754408955574036, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 83 @@ -2422,25 +2422,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, + "completions/clipped_ratio": -4.96875, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1150.0, - "completions/mean_length": 497.54949951171875, - "completions/mean_terminated_length": 489.4908752441406, - "completions/min_length": 222.0, - "completions/min_terminated_length": 222.0, - "entropy": 0.087127685546875, + "completions/max_terminated_length": 1117.0, + "completions/mean_length": 483.5625, + "completions/mean_terminated_length": 467.329833984375, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.110076904296875, "epoch": 0.7664670658682635, "frac_reward_zero_std": 0.0, - "grad_norm": 0.04500949755311012, - "kl": 0.008170127868652344, - "learning_rate": 3.4513926605471504e-06, - "loss": -0.0097, - "num_tokens": 34654848.0, - "reward": 0.563988208770752, - "reward_std": 0.09649153798818588, - "rewards/cross_solution_unittest/mean": 0.563988208770752, - "rewards/cross_solution_unittest/std": 0.17347638309001923, + "grad_norm": 0.03293149545788765, + "kl": 0.0037369728088378906, + "learning_rate": 2.8286819524103657e-06, + "loss": -0.0012, + "num_tokens": 34176409.0, + "reward": 0.8954019546508789, + "reward_std": 0.20441854000091553, + "rewards/cross_solution_unittest_reward/mean": 0.44770099222660065, + "rewards/cross_solution_unittest_reward/std": 0.21769069880247116, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 84 @@ -2451,25 +2451,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1627.0, - "completions/mean_length": 516.9948120117188, - "completions/mean_terminated_length": 508.9869384765625, - "completions/min_length": 236.0, - "completions/min_terminated_length": 236.0, - "entropy": 0.0800933837890625, + "completions/clipped_ratio": -5.0, + "completions/max_length": 937.0, + "completions/max_terminated_length": 937.0, + "completions/mean_length": 485.5442810058594, + "completions/mean_terminated_length": 485.5442810058594, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.0960693359375, "epoch": 0.7755916737952666, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03465050458908081, - "kl": 0.007006645202636719, - "learning_rate": 3.2149058844286796e-06, - "loss": 0.0084, - "num_tokens": 35092086.0, - "reward": 0.577928900718689, - "reward_std": 0.08402758836746216, - "rewards/cross_solution_unittest/mean": 0.5779288411140442, - "rewards/cross_solution_unittest/std": 0.1539035141468048, + "grad_norm": 0.033041298389434814, + "kl": 0.004325389862060547, + "learning_rate": 2.6325886212359496e-06, + "loss": 0.0025, + "num_tokens": 34602482.0, + "reward": 0.9020225405693054, + "reward_std": 0.16266906261444092, + "rewards/cross_solution_unittest_reward/mean": 0.4510112926363945, + "rewards/cross_solution_unittest_reward/std": 0.19898167252540588, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 85 @@ -2480,25 +2480,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, + "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 905.0, - "completions/mean_length": 493.29168701171875, - "completions/mean_terminated_length": 477.1099548339844, - "completions/min_length": 250.0, - "completions/min_terminated_length": 250.0, - "entropy": 0.0883026123046875, + "completions/max_terminated_length": 826.0, + "completions/mean_length": 472.5625, + "completions/mean_terminated_length": 464.43865966796875, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.106231689453125, "epoch": 0.7847162817222697, "frac_reward_zero_std": 0.0, - "grad_norm": 0.040905412286520004, - "kl": 0.008014678955078125, - "learning_rate": 2.9852511229367862e-06, - "loss": -0.0056, - "num_tokens": 35510442.0, - "reward": 0.6279939413070679, - "reward_std": 0.08435385674238205, - "rewards/cross_solution_unittest/mean": 0.6279939413070679, - "rewards/cross_solution_unittest/std": 0.13109418749809265, + "grad_norm": 0.03541736304759979, + "kl": 0.005221366882324219, + "learning_rate": 2.4425042564574186e-06, + "loss": -0.0027, + "num_tokens": 35012690.0, + "reward": 0.8730189204216003, + "reward_std": 0.19061098992824554, + "rewards/cross_solution_unittest_reward/mean": 0.4365094378590584, + "rewards/cross_solution_unittest_reward/std": 0.19948572665452957, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 86 @@ -2509,25 +2509,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, + "completions/clipped_ratio": -4.953125, "completions/max_length": 3584.0, - "completions/max_terminated_length": 2231.0, - "completions/mean_length": 529.7109375, - "completions/mean_terminated_length": 513.7199096679688, - "completions/min_length": 266.0, - "completions/min_terminated_length": 266.0, - "entropy": 0.085418701171875, + "completions/max_terminated_length": 932.0, + "completions/mean_length": 523.3880615234375, + "completions/mean_terminated_length": 499.2887268066406, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.1020660400390625, "epoch": 0.7938408896492729, "frac_reward_zero_std": 0.0, - "grad_norm": 0.04118010401725769, - "kl": 0.007990837097167969, - "learning_rate": 2.7626596189492983e-06, - "loss": -0.0104, - "num_tokens": 35943883.0, - "reward": 0.6178367137908936, - "reward_std": 0.07810844480991364, - "rewards/cross_solution_unittest/mean": 0.6178367137908936, - "rewards/cross_solution_unittest/std": 0.12682530283927917, + "grad_norm": 0.035699885338544846, + "kl": 0.0038251876831054688, + "learning_rate": 2.2585838936091753e-06, + "loss": 0.0055, + "num_tokens": 35446079.0, + "reward": 0.9853703379631042, + "reward_std": 0.1933068484067917, + "rewards/cross_solution_unittest_reward/mean": 0.49268509447574615, + "rewards/cross_solution_unittest_reward/std": 0.2293217033147812, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 87 @@ -2539,24 +2539,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 866.0, - "completions/max_terminated_length": 866.0, - "completions/mean_length": 436.6796875, - "completions/mean_terminated_length": 436.6796875, - "completions/min_length": 217.0, - "completions/min_terminated_length": 217.0, - "entropy": 0.0947723388671875, + "completions/max_length": 949.0, + "completions/max_terminated_length": 949.0, + "completions/mean_length": 436.2578125, + "completions/mean_terminated_length": 436.2578125, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.1071624755859375, "epoch": 0.802965497576276, "frac_reward_zero_std": 0.0, - "grad_norm": 0.05317376181483269, - "kl": 0.009276390075683594, - "learning_rate": 2.5473555032424534e-06, - "loss": 0.0077, - "num_tokens": 36297980.0, - "reward": 0.6129441261291504, - "reward_std": 0.08101849257946014, - "rewards/cross_solution_unittest/mean": 0.6129441261291504, - "rewards/cross_solution_unittest/std": 0.15165738761425018, + "grad_norm": 0.12767620384693146, + "kl": 0.010437488555908203, + "learning_rate": 2.0809775407772505e-06, + "loss": 0.0142, + "num_tokens": 35801002.0, + "reward": 0.9056326746940613, + "reward_std": 0.22697775065898895, + "rewards/cross_solution_unittest_reward/mean": 0.45281632244586945, + "rewards/cross_solution_unittest_reward/std": 0.2281804159283638, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 88 @@ -2568,24 +2568,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 991.0, - "completions/max_terminated_length": 991.0, - "completions/mean_length": 515.4791870117188, - "completions/mean_terminated_length": 515.4791870117188, - "completions/min_length": 264.0, - "completions/min_terminated_length": 264.0, - "entropy": 0.07867431640625, + "completions/max_length": 821.0, + "completions/max_terminated_length": 821.0, + "completions/mean_length": 501.71356201171875, + "completions/mean_terminated_length": 501.71356201171875, + "completions/min_length": 316.0, + "completions/min_terminated_length": 316.0, + "entropy": 0.0912628173828125, "epoch": 0.8120901055032792, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03746452555060387, - "kl": 0.0072307586669921875, - "learning_rate": 2.339555568810221e-06, - "loss": 0.014, - "num_tokens": 36730140.0, - "reward": 0.5748163461685181, - "reward_std": 0.06574384868144989, - "rewards/cross_solution_unittest/mean": 0.5748162865638733, - "rewards/cross_solution_unittest/std": 0.12489957362413406, + "grad_norm": 0.030100375413894653, + "kl": 0.004030466079711914, + "learning_rate": 1.9098300562505266e-06, + "loss": 0.0095, + "num_tokens": 36223060.0, + "reward": 0.8723653554916382, + "reward_std": 0.1536586731672287, + "rewards/cross_solution_unittest_reward/mean": 0.4361826628446579, + "rewards/cross_solution_unittest_reward/std": 0.24371575564146042, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 89 @@ -2596,25 +2596,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1047.0, - "completions/mean_length": 518.2213745117188, - "completions/mean_terminated_length": 502.170166015625, - "completions/min_length": 280.0, - "completions/min_terminated_length": 280.0, - "entropy": 0.07883453369140625, + "completions/clipped_ratio": -5.0, + "completions/max_length": 846.0, + "completions/max_terminated_length": 846.0, + "completions/mean_length": 482.5078125, + "completions/mean_terminated_length": 482.5078125, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.096923828125, "epoch": 0.8212147134302823, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03742819279432297, - "kl": 0.007193565368652344, - "learning_rate": 2.1394690525721275e-06, - "loss": -0.0006, - "num_tokens": 37153593.0, - "reward": 0.5915634632110596, - "reward_std": 0.07696962356567383, - "rewards/cross_solution_unittest/mean": 0.5915634632110596, - "rewards/cross_solution_unittest/std": 0.12713728845119476, + "grad_norm": 0.040627796202898026, + "kl": 0.007803916931152344, + "learning_rate": 1.74528103037226e-06, + "loss": -0.0038, + "num_tokens": 36635343.0, + "reward": 0.7952584028244019, + "reward_std": 0.23465335369110107, + "rewards/cross_solution_unittest_reward/mean": 0.39762915670871735, + "rewards/cross_solution_unittest_reward/std": 0.2743668779730797, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 90 @@ -2625,25 +2625,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 922.0, - "completions/mean_length": 477.1171875, - "completions/mean_terminated_length": 469.0052185058594, - "completions/min_length": 235.0, - "completions/min_terminated_length": 235.0, - "entropy": 0.0898284912109375, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1028.0, + "completions/max_terminated_length": 1028.0, + "completions/mean_length": 482.26824951171875, + "completions/mean_terminated_length": 482.26824951171875, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.1040496826171875, "epoch": 0.8303393213572854, "frac_reward_zero_std": 0.0, - "grad_norm": 0.04124828428030014, - "kl": 0.010035514831542969, - "learning_rate": 1.947297424689414e-06, - "loss": -0.0044, - "num_tokens": 37537306.0, - "reward": 0.6299624443054199, - "reward_std": 0.08996909856796265, - "rewards/cross_solution_unittest/mean": 0.6299623847007751, - "rewards/cross_solution_unittest/std": 0.1566157191991806, + "grad_norm": 0.036647334694862366, + "kl": 0.0043468475341796875, + "learning_rate": 1.587464671688187e-06, + "loss": -0.005, + "num_tokens": 37027198.0, + "reward": 0.8686568140983582, + "reward_std": 0.1460837721824646, + "rewards/cross_solution_unittest_reward/mean": 0.4343283474445343, + "rewards/cross_solution_unittest_reward/std": 0.21033113449811935, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 91 @@ -2656,23 +2656,23 @@ "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.96875, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1312.0, - "completions/mean_length": 511.70574951171875, - "completions/mean_terminated_length": 495.62042236328125, - "completions/min_length": 205.0, - "completions/min_terminated_length": 205.0, - "entropy": 0.0867462158203125, + "completions/max_terminated_length": 941.0, + "completions/mean_length": 507.2317810058594, + "completions/mean_terminated_length": 491.123046875, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "entropy": 0.09820556640625, "epoch": 0.8394639292842886, "frac_reward_zero_std": 0.0, - "grad_norm": 0.04116717353463173, - "kl": 0.009945869445800781, - "learning_rate": 1.7632341857016733e-06, - "loss": 0.0106, - "num_tokens": 37959049.0, - "reward": 0.613093376159668, - "reward_std": 0.07671508938074112, - "rewards/cross_solution_unittest/mean": 0.6130933165550232, - "rewards/cross_solution_unittest/std": 0.09333295375108719, + "grad_norm": 0.033177897334098816, + "kl": 0.004420280456542969, + "learning_rate": 1.436509697484111e-06, + "loss": -0.001, + "num_tokens": 37446527.0, + "reward": 0.8312811851501465, + "reward_std": 0.22770294547080994, + "rewards/cross_solution_unittest_reward/mean": 0.41564057022333145, + "rewards/cross_solution_unittest_reward/std": 0.25115445256233215, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 92 @@ -2685,23 +2685,23 @@ "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.921875, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1230.0, - "completions/mean_length": 576.078125, - "completions/mean_terminated_length": 536.3958129882812, - "completions/min_length": 11.0, - "completions/min_terminated_length": 11.0, - "entropy": 0.07297515869140625, + "completions/max_terminated_length": 1317.0, + "completions/mean_length": 578.09375, + "completions/mean_terminated_length": 538.43798828125, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.086181640625, "epoch": 0.8485885372112917, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03428766503930092, - "kl": 0.007388114929199219, - "learning_rate": 1.587464671688187e-06, - "loss": -0.0183, - "num_tokens": 38399059.0, - "reward": 0.618889331817627, - "reward_std": 0.08409196138381958, - "rewards/cross_solution_unittest/mean": 0.618889331817627, - "rewards/cross_solution_unittest/std": 0.12098271399736404, + "grad_norm": 0.0323200523853302, + "kl": 0.004235267639160156, + "learning_rate": 1.2925392288022299e-06, + "loss": 0.0244, + "num_tokens": 37887771.0, + "reward": 0.7843211889266968, + "reward_std": 0.22614744305610657, + "rewards/cross_solution_unittest_reward/mean": 0.3921606168150902, + "rewards/cross_solution_unittest_reward/std": 0.23859115689992905, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 93 @@ -2714,23 +2714,23 @@ "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 1236.0, - "completions/mean_length": 530.6068115234375, - "completions/mean_terminated_length": 522.6344604492188, + "completions/max_terminated_length": 961.0, + "completions/mean_length": 527.734375, + "completions/mean_terminated_length": 519.7545776367188, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, - "entropy": 0.07482147216796875, + "entropy": 0.089080810546875, "epoch": 0.8577131451382949, "frac_reward_zero_std": 0.0, - "grad_norm": 0.0353349931538105, - "kl": 0.006430149078369141, - "learning_rate": 1.4201658676502294e-06, - "loss": 0.0119, - "num_tokens": 38818392.0, - "reward": 0.658619225025177, - "reward_std": 0.08192513883113861, - "rewards/cross_solution_unittest/mean": 0.6586191654205322, - "rewards/cross_solution_unittest/std": 0.11717738956212997, + "grad_norm": 0.03161383792757988, + "kl": 0.0037050247192382812, + "learning_rate": 1.1556706900218572e-06, + "loss": -0.0029, + "num_tokens": 38306421.0, + "reward": 0.8312766551971436, + "reward_std": 0.25127798318862915, + "rewards/cross_solution_unittest_reward/mean": 0.4156383126974106, + "rewards/cross_solution_unittest_reward/std": 0.27913951128721237, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 94 @@ -2742,24 +2742,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 929.0, - "completions/max_terminated_length": 929.0, - "completions/mean_length": 493.29168701171875, - "completions/mean_terminated_length": 493.29168701171875, - "completions/min_length": 234.0, - "completions/min_terminated_length": 234.0, - "entropy": 0.0816497802734375, + "completions/max_length": 762.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 482.65106201171875, + "completions/mean_terminated_length": 482.65106201171875, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.096649169921875, "epoch": 0.866837753065298, "frac_reward_zero_std": 0.0, - "grad_norm": 0.0350792221724987, - "kl": 0.007628440856933594, - "learning_rate": 1.2615062293021508e-06, - "loss": 0.002, - "num_tokens": 39264744.0, - "reward": 0.6107174158096313, - "reward_std": 0.06959541141986847, - "rewards/cross_solution_unittest/mean": 0.6107173562049866, - "rewards/cross_solution_unittest/std": 0.09727025777101517, + "grad_norm": 0.030079808086156845, + "kl": 0.004782915115356445, + "learning_rate": 1.0260157130864178e-06, + "loss": 0.0004, + "num_tokens": 38748439.0, + "reward": 0.8986494541168213, + "reward_std": 0.20043903589248657, + "rewards/cross_solution_unittest_reward/mean": 0.44932468235492706, + "rewards/cross_solution_unittest_reward/std": 0.1927783116698265, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 95 @@ -2770,25 +2770,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1227.0, - "completions/mean_length": 532.9869995117188, - "completions/mean_terminated_length": 525.0208740234375, - "completions/min_length": 272.0, - "completions/min_terminated_length": 272.0, - "entropy": 0.079864501953125, + "completions/clipped_ratio": -5.0, + "completions/max_length": 878.0, + "completions/max_terminated_length": 878.0, + "completions/mean_length": 512.8932495117188, + "completions/mean_terminated_length": 512.8932495117188, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "entropy": 0.0985107421875, "epoch": 0.8759623609923011, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03818963095545769, - "kl": 0.006740570068359375, - "learning_rate": 1.1116455134507665e-06, - "loss": -0.0028, - "num_tokens": 39659039.0, - "reward": 0.5914241671562195, - "reward_std": 0.07413486391305923, - "rewards/cross_solution_unittest/mean": 0.5914241671562195, - "rewards/cross_solution_unittest/std": 0.14320488274097443, + "grad_norm": 0.035013340413570404, + "kl": 0.004712104797363281, + "learning_rate": 9.036800464548157e-07, + "loss": 0.0024, + "num_tokens": 39133230.0, + "reward": 0.8117542266845703, + "reward_std": 0.20024093985557556, + "rewards/cross_solution_unittest_reward/mean": 0.40587710589170456, + "rewards/cross_solution_unittest_reward/std": 0.2243197336792946, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 96 @@ -2799,25 +2799,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1008.0, - "completions/max_terminated_length": 1008.0, - "completions/mean_length": 460.04168701171875, - "completions/mean_terminated_length": 460.04168701171875, - "completions/min_length": 205.0, - "completions/min_terminated_length": 205.0, - "entropy": 0.0875091552734375, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 811.0, + "completions/mean_length": 455.43231201171875, + "completions/mean_terminated_length": 447.2637023925781, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.1064453125, "epoch": 0.8850869689193043, "frac_reward_zero_std": 0.0, - "grad_norm": 0.041273754090070724, - "kl": 0.007297515869140625, - "learning_rate": 9.707346171337895e-07, - "loss": -0.0054, - "num_tokens": 39981139.0, - "reward": 0.6115624308586121, - "reward_std": 0.07325311750173569, - "rewards/cross_solution_unittest/mean": 0.6115624308586121, - "rewards/cross_solution_unittest/std": 0.09799737483263016, + "grad_norm": 0.03698151558637619, + "kl": 0.004943370819091797, + "learning_rate": 7.887634688515e-07, + "loss": 0.0116, + "num_tokens": 39453516.0, + "reward": 1.0091822147369385, + "reward_std": 0.17214202880859375, + "rewards/cross_solution_unittest_reward/mean": 0.5045911073684692, + "rewards/cross_solution_unittest_reward/std": 0.2308211475610733, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 97 @@ -2829,24 +2829,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1656.0, - "completions/max_terminated_length": 1656.0, - "completions/mean_length": 526.7291870117188, - "completions/mean_terminated_length": 526.7291870117188, - "completions/min_length": 272.0, - "completions/min_terminated_length": 272.0, - "entropy": 0.0795745849609375, + "completions/max_length": 905.0, + "completions/max_terminated_length": 905.0, + "completions/mean_length": 517.7213745117188, + "completions/mean_terminated_length": 517.7213745117188, + "completions/min_length": 332.0, + "completions/min_terminated_length": 332.0, + "entropy": 0.09765625, "epoch": 0.8942115768463074, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03502219542860985, - "kl": 0.006832599639892578, - "learning_rate": 8.389154256793042e-07, - "loss": -0.0011, - "num_tokens": 40425327.0, - "reward": 0.6242746710777283, - "reward_std": 0.06769298017024994, - "rewards/cross_solution_unittest/mean": 0.6242746114730835, - "rewards/cross_solution_unittest/std": 0.11656895279884338, + "grad_norm": 0.0317588746547699, + "kl": 0.004177570343017578, + "learning_rate": 6.813597078854772e-07, + "loss": -0.0053, + "num_tokens": 39895593.0, + "reward": 0.9603371620178223, + "reward_std": 0.18226468563079834, + "rewards/cross_solution_unittest_reward/mean": 0.48016858100891113, + "rewards/cross_solution_unittest_reward/std": 0.2831304222345352, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 98 @@ -2857,25 +2857,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 2244.0, - "completions/mean_length": 509.94793701171875, - "completions/mean_terminated_length": 501.92169189453125, - "completions/min_length": 226.0, - "completions/min_terminated_length": 226.0, - "entropy": 0.0839080810546875, + "completions/clipped_ratio": -5.0, + "completions/max_length": 795.0, + "completions/max_terminated_length": 795.0, + "completions/mean_length": 476.8515625, + "completions/mean_terminated_length": 476.8515625, + "completions/min_length": 175.0, + "completions/min_terminated_length": 175.0, + "entropy": 0.1009063720703125, "epoch": 0.9033361847733106, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03684497997164726, - "kl": 0.009736061096191406, - "learning_rate": 7.163206698392744e-07, - "loss": 0.0016, - "num_tokens": 40804411.0, - "reward": 0.5832025408744812, - "reward_std": 0.08364126086235046, - "rewards/cross_solution_unittest/mean": 0.5832025408744812, - "rewards/cross_solution_unittest/std": 0.15208092331886292, + "grad_norm": 0.03179348632693291, + "kl": 0.004183292388916016, + "learning_rate": 5.815563636047539e-07, + "loss": -0.0026, + "num_tokens": 40260864.0, + "reward": 0.9521991014480591, + "reward_std": 0.2101556956768036, + "rewards/cross_solution_unittest_reward/mean": 0.47609956562519073, + "rewards/cross_solution_unittest_reward/std": 0.225713312625885, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 99 @@ -2887,24 +2887,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 859.0, - "completions/max_terminated_length": 859.0, - "completions/mean_length": 454.2734375, - "completions/mean_terminated_length": 454.2734375, - "completions/min_length": 188.0, - "completions/min_terminated_length": 188.0, - "entropy": 0.094268798828125, + "completions/max_length": 798.0, + "completions/max_terminated_length": 798.0, + "completions/mean_length": 457.1015625, + "completions/mean_terminated_length": 457.1015625, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "entropy": 0.1097564697265625, "epoch": 0.9124607927003137, "frac_reward_zero_std": 0.0, - "grad_norm": 0.03873429447412491, - "kl": 0.009373664855957031, - "learning_rate": 6.030737921409169e-07, - "loss": 0.0099, - "num_tokens": 41217760.0, - "reward": 0.6278964281082153, - "reward_std": 0.07669099420309067, - "rewards/cross_solution_unittest/mean": 0.6278963685035706, - "rewards/cross_solution_unittest/std": 0.09835278242826462, + "grad_norm": 0.03737381473183632, + "kl": 0.0048885345458984375, + "learning_rate": 4.894348370484648e-07, + "loss": 0.0142, + "num_tokens": 40678271.0, + "reward": 0.8324875235557556, + "reward_std": 0.20148925483226776, + "rewards/cross_solution_unittest_reward/mean": 0.416243739426136, + "rewards/cross_solution_unittest_reward/std": 0.25171586126089096, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 100 @@ -2915,25 +2915,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, + "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, - "completions/max_terminated_length": 896.0, - "completions/mean_length": 502.94793701171875, - "completions/mean_terminated_length": 486.8167724609375, - "completions/min_length": 246.0, - "completions/min_terminated_length": 246.0, - "entropy": 0.1176605224609375, + "completions/max_terminated_length": 1004.0, + "completions/mean_length": 483.8359375, + "completions/mean_terminated_length": 475.74151611328125, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.09332275390625, "epoch": 0.9215854006273168, - "frac_reward_zero_std": 0.1666666716337204, - "grad_norm": 0.05346548557281494, - "kl": 0.008120536804199219, - "learning_rate": 4.992888225905467e-07, - "loss": -0.0016, - "num_tokens": 41613372.0, - "reward": 1.239593744277954, - "reward_std": 0.3236619830131531, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2395936250686646, - "rewards/cross_solution_unittest_reward_v2/std": 0.6752111911773682, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.030489500612020493, + "kl": 0.004380226135253906, + "learning_rate": 4.0507026385502747e-07, + "loss": 0.0107, + "num_tokens": 41066632.0, + "reward": 0.9275814890861511, + "reward_std": 0.1952568143606186, + "rewards/cross_solution_unittest_reward/mean": 0.46379074454307556, + "rewards/cross_solution_unittest_reward/std": 0.20886488258838654, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 101 @@ -2944,25 +2944,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1002.0, - "completions/mean_length": 517.1901245117188, - "completions/mean_terminated_length": 509.1827697753906, - "completions/min_length": 198.0, - "completions/min_terminated_length": 198.0, - "entropy": 0.1147613525390625, + "completions/clipped_ratio": -5.0, + "completions/max_length": 876.0, + "completions/max_terminated_length": 876.0, + "completions/mean_length": 490.8802185058594, + "completions/mean_terminated_length": 490.8802185058594, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.0925445556640625, "epoch": 0.93071000855432, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.03924965858459473, - "kl": 0.0075531005859375, - "learning_rate": 1.3120334456984871e-05, - "loss": -0.0036, - "num_tokens": 42049189.0, - "reward": 1.295283555984497, - "reward_std": 0.30770766735076904, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2952834367752075, - "rewards/cross_solution_unittest_reward_v2/std": 0.5776005387306213, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02890767715871334, + "kl": 0.004089832305908203, + "learning_rate": 3.2853145298042954e-07, + "loss": 0.0088, + "num_tokens": 41490450.0, + "reward": 1.0289883613586426, + "reward_std": 0.23402146995067596, + "rewards/cross_solution_unittest_reward/mean": 0.5144941955804825, + "rewards/cross_solution_unittest_reward/std": 0.20954615622758865, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 102 @@ -2973,25 +2973,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1168.0, - "completions/mean_length": 485.9192810058594, - "completions/mean_terminated_length": 469.6989440917969, - "completions/min_length": 231.0, - "completions/min_terminated_length": 231.0, - "entropy": 0.1226806640625, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1279.0, + "completions/max_terminated_length": 1279.0, + "completions/mean_length": 471.7421875, + "completions/mean_terminated_length": 471.7421875, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.100799560546875, "epoch": 0.939834616481323, "frac_reward_zero_std": 0.0, - "grad_norm": 0.11646046489477158, - "kl": 0.014455795288085938, - "learning_rate": 1.296920375328275e-05, - "loss": 0.0243, - "num_tokens": 42423846.0, - "reward": 1.248105525970459, - "reward_std": 0.3160620927810669, - "rewards/cross_solution_unittest_reward_v2/mean": 1.248105525970459, - "rewards/cross_solution_unittest_reward_v2/std": 0.6170075535774231, + "grad_norm": 0.039594829082489014, + "kl": 0.005157470703125, + "learning_rate": 2.5988083057666534e-07, + "loss": 0.0164, + "num_tokens": 41862519.0, + "reward": 1.0344469547271729, + "reward_std": 0.2174767255783081, + "rewards/cross_solution_unittest_reward/mean": 0.5172234624624252, + "rewards/cross_solution_unittest_reward/std": 0.2513153925538063, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 103 @@ -3003,24 +3003,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1257.0, - "completions/max_terminated_length": 1257.0, - "completions/mean_length": 503.4505310058594, - "completions/mean_terminated_length": 503.4505310058594, - "completions/min_length": 263.0, - "completions/min_terminated_length": 263.0, - "entropy": 0.118927001953125, + "completions/max_length": 833.0, + "completions/max_terminated_length": 833.0, + "completions/mean_length": 489.171875, + "completions/mean_terminated_length": 489.171875, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.0948944091796875, "epoch": 0.9489592244083263, "frac_reward_zero_std": 0.0, - "grad_norm": 0.05845910310745239, - "kl": 0.010087013244628906, - "learning_rate": 1.2817325568414299e-05, - "loss": 0.0099, - "num_tokens": 42843235.0, - "reward": 1.1288866996765137, - "reward_std": 0.2721058130264282, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1288866996765137, - "rewards/cross_solution_unittest_reward_v2/std": 0.5756978988647461, + "grad_norm": 0.04726433753967285, + "kl": 0.005943775177001953, + "learning_rate": 1.9917438907606556e-07, + "loss": 0.0073, + "num_tokens": 42276585.0, + "reward": 0.8814212679862976, + "reward_std": 0.1902289092540741, + "rewards/cross_solution_unittest_reward/mean": 0.4407106339931488, + "rewards/cross_solution_unittest_reward/std": 0.1961411088705063, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 104 @@ -3031,25 +3031,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 3512.0, - "completions/mean_length": 481.1770935058594, - "completions/mean_terminated_length": 473.0757141113281, - "completions/min_length": 248.0, - "completions/min_terminated_length": 248.0, - "entropy": 0.12969970703125, + "completions/clipped_ratio": -5.0, + "completions/max_length": 737.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 453.78125, + "completions/mean_terminated_length": 453.78125, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "entropy": 0.10247802734375, "epoch": 0.9580838323353293, "frac_reward_zero_std": 0.0, - "grad_norm": 0.058683473616838455, - "kl": 0.008669853210449219, - "learning_rate": 1.266473813690035e-05, - "loss": 0.0035, - "num_tokens": 43213319.0, - "reward": 0.9867174625396729, - "reward_std": 0.4156356751918793, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9867174029350281, - "rewards/cross_solution_unittest_reward_v2/std": 0.6481515765190125, + "grad_norm": 0.0373138003051281, + "kl": 0.004960536956787109, + "learning_rate": 1.464616415230702e-07, + "loss": -0.0085, + "num_tokens": 42634949.0, + "reward": 0.7063878774642944, + "reward_std": 0.22609077394008636, + "rewards/cross_solution_unittest_reward/mean": 0.3531939312815666, + "rewards/cross_solution_unittest_reward/std": 0.23600679636001587, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 105 @@ -3061,24 +3061,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 860.0, - "completions/max_terminated_length": 860.0, - "completions/mean_length": 520.6588745117188, - "completions/mean_terminated_length": 520.6588745117188, - "completions/min_length": 277.0, - "completions/min_terminated_length": 277.0, - "entropy": 0.1167144775390625, + "completions/max_length": 814.0, + "completions/max_terminated_length": 814.0, + "completions/mean_length": 514.3932495117188, + "completions/mean_terminated_length": 514.3932495117188, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "entropy": 0.0894622802734375, "epoch": 0.9672084402623324, "frac_reward_zero_std": 0.0, - "grad_norm": 0.047805868089199066, - "kl": 0.007547855377197266, - "learning_rate": 1.2511479871810792e-05, - "loss": -0.0082, - "num_tokens": 43639116.0, - "reward": 1.076136827468872, - "reward_std": 0.32517102360725403, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0761367082595825, - "rewards/cross_solution_unittest_reward_v2/std": 0.6100772023200989, + "grad_norm": 0.028397122398018837, + "kl": 0.004255771636962891, + "learning_rate": 1.0178558119067316e-07, + "loss": -0.008, + "num_tokens": 43057844.0, + "reward": 0.8395287990570068, + "reward_std": 0.23434078693389893, + "rewards/cross_solution_unittest_reward/mean": 0.4197644293308258, + "rewards/cross_solution_unittest_reward/std": 0.25368085503578186, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 106 @@ -3089,25 +3089,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1043.0, - "completions/mean_length": 497.6302185058594, - "completions/mean_terminated_length": 489.5718078613281, - "completions/min_length": 252.0, - "completions/min_terminated_length": 252.0, - "entropy": 0.1410064697265625, + "completions/clipped_ratio": -5.0, + "completions/max_length": 2142.0, + "completions/max_terminated_length": 2142.0, + "completions/mean_length": 488.59375, + "completions/mean_terminated_length": 488.59375, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.10888671875, "epoch": 0.9763330481893356, "frac_reward_zero_std": 0.0, - "grad_norm": 0.04889044165611267, - "kl": 0.008774757385253906, - "learning_rate": 1.2357589355094275e-05, - "loss": 0.0033, - "num_tokens": 44045918.0, - "reward": 1.227220058441162, - "reward_std": 0.322670578956604, - "rewards/cross_solution_unittest_reward_v2/mean": 1.227220058441162, - "rewards/cross_solution_unittest_reward_v2/std": 0.5605356693267822, + "grad_norm": 0.034389153122901917, + "kl": 0.004794120788574219, + "learning_rate": 6.51826465144978e-08, + "loss": 0.003, + "num_tokens": 43460008.0, + "reward": 0.9071213603019714, + "reward_std": 0.1905367076396942, + "rewards/cross_solution_unittest_reward/mean": 0.4535607025027275, + "rewards/cross_solution_unittest_reward/std": 0.2374211624264717, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 107 @@ -3119,24 +3119,24 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 1097.0, - "completions/max_terminated_length": 1097.0, - "completions/mean_length": 526.5546875, - "completions/mean_terminated_length": 526.5546875, - "completions/min_length": 213.0, - "completions/min_terminated_length": 213.0, - "entropy": 0.1137237548828125, + "completions/max_length": 1084.0, + "completions/max_terminated_length": 1084.0, + "completions/mean_length": 535.3229370117188, + "completions/mean_terminated_length": 535.3229370117188, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.0862884521484375, "epoch": 0.9854576561163387, "frac_reward_zero_std": 0.0, - "grad_norm": 0.036034442484378815, - "kl": 0.007466316223144531, - "learning_rate": 1.2203105327865407e-05, - "loss": 0.001, - "num_tokens": 44486883.0, - "reward": 0.9676321744918823, - "reward_std": 0.33799371123313904, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9676321148872375, - "rewards/cross_solution_unittest_reward_v2/std": 0.5765103101730347, + "grad_norm": 0.029783420264720917, + "kl": 0.0033812522888183594, + "learning_rate": 3.668269137308666e-08, + "loss": -0.0022, + "num_tokens": 43903588.0, + "reward": 1.0605829954147339, + "reward_std": 0.17206411063671112, + "rewards/cross_solution_unittest_reward/mean": 0.5302915126085281, + "rewards/cross_solution_unittest_reward/std": 0.2089696228504181, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 108 @@ -3148,2680 +3148,41 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, - "completions/max_length": 947.0, - "completions/max_terminated_length": 947.0, - "completions/mean_length": 487.36981201171875, - "completions/mean_terminated_length": 487.36981201171875, - "completions/min_length": 229.0, - "completions/min_terminated_length": 229.0, - "entropy": 0.1241607666015625, + "completions/max_length": 940.0, + "completions/max_terminated_length": 940.0, + "completions/mean_length": 456.51824951171875, + "completions/mean_terminated_length": 456.51824951171875, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "entropy": 0.1035308837890625, "epoch": 0.9945822640433419, "frac_reward_zero_std": 0.0, - "grad_norm": 0.038370098918676376, - "kl": 0.007198333740234375, - "learning_rate": 1.2048066680651908e-05, - "loss": -0.0119, - "num_tokens": 44867377.0, - "reward": 1.2263835668563843, - "reward_std": 0.365733802318573, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2263835668563843, - "rewards/cross_solution_unittest_reward_v2/std": 0.569444477558136, + "grad_norm": 0.034084659069776535, + "kl": 0.004221439361572266, + "learning_rate": 1.630896073864352e-08, + "loss": 0.0052, + "num_tokens": 44271795.0, + "reward": 1.0674467086791992, + "reward_std": 0.20672927796840668, + "rewards/cross_solution_unittest_reward/mean": 0.5337233692407608, + "rewards/cross_solution_unittest_reward/std": 0.2225872203707695, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 109 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 2454.0, - "completions/mean_length": 525.5963745117188, - "completions/mean_terminated_length": 517.6109619140625, - "completions/min_length": 250.0, - "completions/min_terminated_length": 250.0, - "entropy": 0.146697998046875, - "epoch": 1.009124607927003, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.04595029726624489, - "kl": 0.008303642272949219, - "learning_rate": 1.1892512443604103e-05, - "loss": 0.015, - "num_tokens": 45262062.0, - "reward": 1.23362398147583, - "reward_std": 0.35362911224365234, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2336238622665405, - "rewards/cross_solution_unittest_reward_v2/std": 0.6734405159950256, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 110 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 881.0, - "completions/max_terminated_length": 881.0, - "completions/mean_length": 513.4713745117188, - "completions/mean_terminated_length": 513.4713745117188, - "completions/min_length": 254.0, - "completions/min_terminated_length": 254.0, - "entropy": 0.11639404296875, - "epoch": 1.0182492158540062, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04151897132396698, - "kl": 0.006669044494628906, - "learning_rate": 1.1736481776669307e-05, - "loss": 0.0063, - "num_tokens": 45649883.0, - "reward": 1.3678274154663086, - "reward_std": 0.3299106955528259, - "rewards/cross_solution_unittest_reward_v2/mean": 1.3678274154663086, - "rewards/cross_solution_unittest_reward_v2/std": 0.4770379364490509, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 111 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1070.0, - "completions/max_terminated_length": 1070.0, - "completions/mean_length": 485.32293701171875, - "completions/mean_terminated_length": 485.32293701171875, - "completions/min_length": 254.0, - "completions/min_terminated_length": 254.0, - "entropy": 0.1253814697265625, - "epoch": 1.0273738237810095, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03716113045811653, - "kl": 0.006839275360107422, - "learning_rate": 1.15800139597335e-05, - "loss": -0.0067, - "num_tokens": 46045111.0, - "reward": 1.2375826835632324, - "reward_std": 0.28724658489227295, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2375826835632324, - "rewards/cross_solution_unittest_reward_v2/std": 0.5458410382270813, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 112 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 917.0, - "completions/mean_length": 484.59375, - "completions/mean_terminated_length": 476.5013122558594, - "completions/min_length": 250.0, - "completions/min_terminated_length": 250.0, - "entropy": 0.1302490234375, - "epoch": 1.0364984317080126, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04643886908888817, - "kl": 0.008426666259765625, - "learning_rate": 1.1423148382732854e-05, - "loss": 0.0125, - "num_tokens": 46422459.0, - "reward": 1.172203540802002, - "reward_std": 0.3621938228607178, - "rewards/cross_solution_unittest_reward_v2/mean": 1.172203540802002, - "rewards/cross_solution_unittest_reward_v2/std": 0.5624293684959412, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 113 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 964.0, - "completions/max_terminated_length": 964.0, - "completions/mean_length": 484.49481201171875, - "completions/mean_terminated_length": 484.49481201171875, - "completions/min_length": 261.0, - "completions/min_terminated_length": 261.0, - "entropy": 0.1347198486328125, - "epoch": 1.0456230396350157, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.05084635317325592, - "kl": 0.007710456848144531, - "learning_rate": 1.1265924535737494e-05, - "loss": 0.0041, - "num_tokens": 46808969.0, - "reward": 1.2459449768066406, - "reward_std": 0.3194722533226013, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2459449768066406, - "rewards/cross_solution_unittest_reward_v2/std": 0.7003458142280579, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 114 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1065.0, - "completions/max_terminated_length": 1065.0, - "completions/mean_length": 536.203125, - "completions/mean_terminated_length": 536.203125, - "completions/min_length": 227.0, - "completions/min_terminated_length": 227.0, - "entropy": 0.11529541015625, - "epoch": 1.0547476475620188, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.04990876838564873, - "kl": 0.007251739501953125, - "learning_rate": 1.1108381999010111e-05, - "loss": -0.0054, - "num_tokens": 47246671.0, - "reward": 1.2841346263885498, - "reward_std": 0.3121691942214966, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2841345071792603, - "rewards/cross_solution_unittest_reward_v2/std": 0.5919288992881775, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 115 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 2402.0, - "completions/max_terminated_length": 2402.0, - "completions/mean_length": 487.3411560058594, - "completions/mean_terminated_length": 487.3411560058594, - "completions/min_length": 221.0, - "completions/min_terminated_length": 221.0, - "entropy": 0.130340576171875, - "epoch": 1.0638722554890219, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.043775659054517746, - "kl": 0.007085323333740234, - "learning_rate": 1.0950560433041825e-05, - "loss": 0.0001, - "num_tokens": 47674210.0, - "reward": 1.2633024454116821, - "reward_std": 0.31659257411956787, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2633024454116821, - "rewards/cross_solution_unittest_reward_v2/std": 0.6023856997489929, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 116 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1241.0, - "completions/max_terminated_length": 1241.0, - "completions/mean_length": 549.7578125, - "completions/mean_terminated_length": 549.7578125, - "completions/min_length": 266.0, - "completions/min_terminated_length": 266.0, - "entropy": 0.11376953125, - "epoch": 1.0729968634160252, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03854307904839516, - "kl": 0.0073413848876953125, - "learning_rate": 1.0792499568567885e-05, - "loss": -0.0048, - "num_tokens": 48104741.0, - "reward": 0.9348183870315552, - "reward_std": 0.4515117406845093, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9348183274269104, - "rewards/cross_solution_unittest_reward_v2/std": 0.6043286323547363, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 117 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1013.0, - "completions/max_terminated_length": 1013.0, - "completions/mean_length": 522.0078125, - "completions/mean_terminated_length": 522.0078125, - "completions/min_length": 259.0, - "completions/min_terminated_length": 259.0, - "entropy": 0.1197052001953125, - "epoch": 1.0821214713430283, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.037361111491918564, - "kl": 0.006680488586425781, - "learning_rate": 1.0634239196565646e-05, - "loss": 0.0022, - "num_tokens": 48556936.0, - "reward": 1.0405426025390625, - "reward_std": 0.3242799937725067, - "rewards/cross_solution_unittest_reward_v2/mean": 1.040542483329773, - "rewards/cross_solution_unittest_reward_v2/std": 0.7007584571838379, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 118 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 3440.0, - "completions/mean_length": 503.0234375, - "completions/mean_terminated_length": 494.9791259765625, - "completions/min_length": 233.0, - "completions/min_terminated_length": 233.0, - "entropy": 0.115234375, - "epoch": 1.0912460792700314, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03401403874158859, - "kl": 0.007243156433105469, - "learning_rate": 1.0475819158237426e-05, - "loss": -0.0155, - "num_tokens": 48951313.0, - "reward": 1.0603630542755127, - "reward_std": 0.37478360533714294, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0603629350662231, - "rewards/cross_solution_unittest_reward_v2/std": 0.6648305058479309, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 119 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1275.0, - "completions/mean_length": 506.8046875, - "completions/mean_terminated_length": 490.6937255859375, - "completions/min_length": 241.0, - "completions/min_terminated_length": 241.0, - "entropy": 0.1160888671875, - "epoch": 1.1003706871970345, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.05071406066417694, - "kl": 0.007695198059082031, - "learning_rate": 1.031727933498068e-05, - "loss": 0.0035, - "num_tokens": 49367206.0, - "reward": 1.030238389968872, - "reward_std": 0.34741950035095215, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0302382707595825, - "rewards/cross_solution_unittest_reward_v2/std": 0.6187939047813416, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 120 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1313.0, - "completions/mean_length": 519.7734375, - "completions/mean_terminated_length": 503.7303771972656, - "completions/min_length": 218.0, - "completions/min_terminated_length": 218.0, - "entropy": 0.1249237060546875, - "epoch": 1.1094952951240376, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04768640175461769, - "kl": 0.007404327392578125, - "learning_rate": 1.015865963834808e-05, - "loss": 0.0238, - "num_tokens": 49793767.0, - "reward": 1.1717848777770996, - "reward_std": 0.33733150362968445, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1717848777770996, - "rewards/cross_solution_unittest_reward_v2/std": 0.5893012285232544, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 121 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 939.0, - "completions/max_terminated_length": 939.0, - "completions/mean_length": 518.1693115234375, - "completions/mean_terminated_length": 518.1693115234375, - "completions/min_length": 250.0, - "completions/min_terminated_length": 250.0, - "entropy": 0.1182403564453125, - "epoch": 1.1186199030510409, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04382634535431862, - "kl": 0.008069992065429688, - "learning_rate": 1e-05, - "loss": 0.0067, - "num_tokens": 50227160.0, - "reward": 1.1495325565338135, - "reward_std": 0.3532819151878357, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1495325565338135, - "rewards/cross_solution_unittest_reward_v2/std": 0.5747461318969727, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 122 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 840.0, - "completions/mean_length": 531.2943115234375, - "completions/mean_terminated_length": 523.3237915039062, - "completions/min_length": 248.0, - "completions/min_terminated_length": 248.0, - "entropy": 0.1152801513671875, - "epoch": 1.127744510978044, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04340718686580658, - "kl": 0.0068569183349609375, - "learning_rate": 9.841340361651921e-06, - "loss": 0.0031, - "num_tokens": 50650473.0, - "reward": 1.2662526369094849, - "reward_std": 0.31315380334854126, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2662526369094849, - "rewards/cross_solution_unittest_reward_v2/std": 0.4288769066333771, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 123 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 3260.0, - "completions/mean_length": 536.5078125, - "completions/mean_terminated_length": 520.5523681640625, - "completions/min_length": 210.0, - "completions/min_terminated_length": 210.0, - "entropy": 0.1373748779296875, - "epoch": 1.136869118905047, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.0416567400097847, - "kl": 0.007275581359863281, - "learning_rate": 9.682720665019325e-06, - "loss": 0.0127, - "num_tokens": 51053556.0, - "reward": 1.2474865913391113, - "reward_std": 0.3162327706813812, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2474865913391113, - "rewards/cross_solution_unittest_reward_v2/std": 0.5672041177749634, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 124 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1052.0, - "completions/mean_length": 537.8828125, - "completions/mean_terminated_length": 529.9295043945312, - "completions/min_length": 241.0, - "completions/min_terminated_length": 241.0, - "entropy": 0.1173095703125, - "epoch": 1.1459937268320501, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03596977889537811, - "kl": 0.006232261657714844, - "learning_rate": 9.524180841762577e-06, - "loss": -0.0056, - "num_tokens": 51489183.0, - "reward": 0.9407374262809753, - "reward_std": 0.3525381088256836, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9407374262809753, - "rewards/cross_solution_unittest_reward_v2/std": 0.6241452693939209, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 125 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1106.0, - "completions/mean_length": 490.0364685058594, - "completions/mean_terminated_length": 473.83770751953125, - "completions/min_length": 212.0, - "completions/min_terminated_length": 212.0, - "entropy": 0.139862060546875, - "epoch": 1.1551183347590532, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.047817353159189224, - "kl": 0.008927345275878906, - "learning_rate": 9.365760803434356e-06, - "loss": -0.0159, - "num_tokens": 51877333.0, - "reward": 1.087019920349121, - "reward_std": 0.3703293204307556, - "rewards/cross_solution_unittest_reward_v2/mean": 1.087019920349121, - "rewards/cross_solution_unittest_reward_v2/std": 0.6498270034790039, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 126 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1585.0, - "completions/mean_length": 525.1927490234375, - "completions/mean_terminated_length": 509.1780090332031, - "completions/min_length": 229.0, - "completions/min_terminated_length": 229.0, - "entropy": 0.1201629638671875, - "epoch": 1.1642429426860565, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.040600650012493134, - "kl": 0.007849693298339844, - "learning_rate": 9.207500431432115e-06, - "loss": 0.0159, - "num_tokens": 52291071.0, - "reward": 1.3082804679870605, - "reward_std": 0.264370858669281, - "rewards/cross_solution_unittest_reward_v2/mean": 1.3082804679870605, - "rewards/cross_solution_unittest_reward_v2/std": 0.613860547542572, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 127 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1150.0, - "completions/max_terminated_length": 1150.0, - "completions/mean_length": 493.1640625, - "completions/mean_terminated_length": 493.1640625, - "completions/min_length": 295.0, - "completions/min_terminated_length": 295.0, - "entropy": 0.1285552978515625, - "epoch": 1.1733675506130596, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04904382675886154, - "kl": 0.008134841918945312, - "learning_rate": 9.049439566958176e-06, - "loss": 0.0001, - "num_tokens": 52696302.0, - "reward": 1.1252853870391846, - "reward_std": 0.359048455953598, - "rewards/cross_solution_unittest_reward_v2/mean": 1.125285267829895, - "rewards/cross_solution_unittest_reward_v2/std": 0.5935911536216736, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 128 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1205.0, - "completions/max_terminated_length": 1205.0, - "completions/mean_length": 496.83856201171875, - "completions/mean_terminated_length": 496.83856201171875, - "completions/min_length": 227.0, - "completions/min_terminated_length": 227.0, - "entropy": 0.1231231689453125, - "epoch": 1.1824921585400627, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.0705753043293953, - "kl": 0.008640289306640625, - "learning_rate": 8.89161800098989e-06, - "loss": -0.0076, - "num_tokens": 53135584.0, - "reward": 1.0311517715454102, - "reward_std": 0.4429343640804291, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0311518907546997, - "rewards/cross_solution_unittest_reward_v2/std": 0.5216948390007019, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 129 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1303.0, - "completions/max_terminated_length": 1303.0, - "completions/mean_length": 527.7239990234375, - "completions/mean_terminated_length": 527.7239990234375, - "completions/min_length": 278.0, - "completions/min_terminated_length": 278.0, - "entropy": 0.121826171875, - "epoch": 1.1916167664670658, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.06737584620714188, - "kl": 0.007891654968261719, - "learning_rate": 8.734075464262507e-06, - "loss": -0.0088, - "num_tokens": 53578782.0, - "reward": 1.2934560775756836, - "reward_std": 0.38895338773727417, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2934560775756836, - "rewards/cross_solution_unittest_reward_v2/std": 0.6111608147621155, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 130 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1563.0, - "completions/mean_length": 506.6224060058594, - "completions/mean_terminated_length": 498.58746337890625, - "completions/min_length": 232.0, - "completions/min_terminated_length": 232.0, - "entropy": 0.12445068359375, - "epoch": 1.200741374394069, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.045849885791540146, - "kl": 0.0070514678955078125, - "learning_rate": 8.576851617267151e-06, - "loss": 0.0103, - "num_tokens": 53993309.0, - "reward": 1.1834197044372559, - "reward_std": 0.3947240710258484, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1834198236465454, - "rewards/cross_solution_unittest_reward_v2/std": 0.6363571286201477, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 131 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1999.0, - "completions/max_terminated_length": 1999.0, - "completions/mean_length": 470.9427185058594, - "completions/mean_terminated_length": 470.9427185058594, - "completions/min_length": 211.0, - "completions/min_terminated_length": 211.0, - "entropy": 0.139251708984375, - "epoch": 1.2098659823210722, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.042013030499219894, - "kl": 0.006268024444580078, - "learning_rate": 8.419986040266502e-06, - "loss": -0.017, - "num_tokens": 54368567.0, - "reward": 1.1750876903533936, - "reward_std": 0.3625323176383972, - "rewards/cross_solution_unittest_reward_v2/mean": 1.175087571144104, - "rewards/cross_solution_unittest_reward_v2/std": 0.5701344013214111, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 132 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.9375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 2921.0, - "completions/mean_length": 524.7213745117188, - "completions/mean_terminated_length": 492.5184326171875, - "completions/min_length": 235.0, - "completions/min_terminated_length": 235.0, - "entropy": 0.1358795166015625, - "epoch": 1.2189905902480753, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.04599786549806595, - "kl": 0.0081329345703125, - "learning_rate": 8.263518223330698e-06, - "loss": 0.0152, - "num_tokens": 54786636.0, - "reward": 0.9693597555160522, - "reward_std": 0.3661820888519287, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9693596959114075, - "rewards/cross_solution_unittest_reward_v2/std": 0.7128695249557495, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 133 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 2079.0, - "completions/mean_length": 546.4557495117188, - "completions/mean_terminated_length": 538.5248413085938, - "completions/min_length": 273.0, - "completions/min_terminated_length": 273.0, - "entropy": 0.12384033203125, - "epoch": 1.2281151981750784, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.039366792887449265, - "kl": 0.006062030792236328, - "learning_rate": 8.107487556395902e-06, - "loss": 0.0244, - "num_tokens": 55285427.0, - "reward": 1.1240730285644531, - "reward_std": 0.28759288787841797, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1240731477737427, - "rewards/cross_solution_unittest_reward_v2/std": 0.4645106792449951, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 134 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1047.0, - "completions/mean_length": 525.2943115234375, - "completions/mean_terminated_length": 517.30810546875, - "completions/min_length": 269.0, - "completions/min_terminated_length": 269.0, - "entropy": 0.121826171875, - "epoch": 1.2372398061020815, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03594588860869408, - "kl": 0.005981922149658203, - "learning_rate": 7.951933319348095e-06, - "loss": -0.0078, - "num_tokens": 55699524.0, - "reward": 1.249282956123352, - "reward_std": 0.339665025472641, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2492828369140625, - "rewards/cross_solution_unittest_reward_v2/std": 0.49411383271217346, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 135 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 794.0, - "completions/max_terminated_length": 794.0, - "completions/mean_length": 475.72918701171875, - "completions/mean_terminated_length": 475.72918701171875, - "completions/min_length": 258.0, - "completions/min_terminated_length": 258.0, - "entropy": 0.136016845703125, - "epoch": 1.2463644140290846, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03808057680726051, - "kl": 0.007435798645019531, - "learning_rate": 7.796894672134594e-06, - "loss": 0.0013, - "num_tokens": 56118892.0, - "reward": 0.9834015369415283, - "reward_std": 0.3272141218185425, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9834015965461731, - "rewards/cross_solution_unittest_reward_v2/std": 0.5538015961647034, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 136 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1215.0, - "completions/max_terminated_length": 1215.0, - "completions/mean_length": 508.171875, - "completions/mean_terminated_length": 508.171875, - "completions/min_length": 178.0, - "completions/min_terminated_length": 178.0, - "entropy": 0.1191864013671875, - "epoch": 1.255489021956088, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.038051728159189224, - "kl": 0.00672149658203125, - "learning_rate": 7.642410644905726e-06, - "loss": 0.0007, - "num_tokens": 56556198.0, - "reward": 1.106317400932312, - "reward_std": 0.26877740025520325, - "rewards/cross_solution_unittest_reward_v2/mean": 1.106317400932312, - "rewards/cross_solution_unittest_reward_v2/std": 0.6404877305030823, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 137 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1242.0, - "completions/mean_length": 498.6796875, - "completions/mean_terminated_length": 490.6240234375, - "completions/min_length": 250.0, - "completions/min_terminated_length": 250.0, - "entropy": 0.1317291259765625, - "epoch": 1.264613629883091, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04010302945971489, - "kl": 0.007061958312988281, - "learning_rate": 7.488520128189209e-06, - "loss": 0.01, - "num_tokens": 56945147.0, - "reward": 0.9137018918991089, - "reward_std": 0.40073394775390625, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9137018322944641, - "rewards/cross_solution_unittest_reward_v2/std": 0.6254523992538452, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 138 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 832.0, - "completions/mean_length": 481.796875, - "completions/mean_terminated_length": 465.55499267578125, - "completions/min_length": 243.0, - "completions/min_terminated_length": 243.0, - "entropy": 0.1410675048828125, - "epoch": 1.273738237810094, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.05836150795221329, - "kl": 0.008604049682617188, - "learning_rate": 7.335261863099652e-06, - "loss": -0.0028, - "num_tokens": 57333933.0, - "reward": 1.2862873077392578, - "reward_std": 0.46950000524520874, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2862873077392578, - "rewards/cross_solution_unittest_reward_v2/std": 0.692533016204834, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 139 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 803.0, - "completions/max_terminated_length": 803.0, - "completions/mean_length": 482.4765625, - "completions/mean_terminated_length": 482.4765625, - "completions/min_length": 198.0, - "completions/min_terminated_length": 198.0, - "entropy": 0.12261962890625, - "epoch": 1.2828628457370972, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.05154620110988617, - "kl": 0.0067348480224609375, - "learning_rate": 7.182674431585703e-06, - "loss": -0.0104, - "num_tokens": 57707812.0, - "reward": 0.9646722674369812, - "reward_std": 0.3492862582206726, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9646722674369812, - "rewards/cross_solution_unittest_reward_v2/std": 0.5699427723884583, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 140 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1296.0, - "completions/max_terminated_length": 1296.0, - "completions/mean_length": 506.0546875, - "completions/mean_terminated_length": 506.0546875, - "completions/min_length": 283.0, - "completions/min_terminated_length": 283.0, - "entropy": 0.124114990234375, - "epoch": 1.2919874536641003, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.04220805689692497, - "kl": 0.006176948547363281, - "learning_rate": 7.0307962467172555e-06, - "loss": -0.0029, - "num_tokens": 58127593.0, - "reward": 1.127100944519043, - "reward_std": 0.29102930426597595, - "rewards/cross_solution_unittest_reward_v2/mean": 1.127100944519043, - "rewards/cross_solution_unittest_reward_v2/std": 0.6249386668205261, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 141 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1140.0, - "completions/mean_length": 551.0130615234375, - "completions/mean_terminated_length": 535.1334838867188, - "completions/min_length": 301.0, - "completions/min_terminated_length": 301.0, - "entropy": 0.1186676025390625, - "epoch": 1.3011120615911036, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.0404638834297657, - "kl": 0.006728649139404297, - "learning_rate": 6.87966554301513e-06, - "loss": 0.0037, - "num_tokens": 58558150.0, - "reward": 1.0898345708847046, - "reward_std": 0.45830807089805603, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0898345708847046, - "rewards/cross_solution_unittest_reward_v2/std": 0.5992898344993591, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 142 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1072.0, - "completions/mean_length": 520.9036865234375, - "completions/mean_terminated_length": 504.8664855957031, - "completions/min_length": 237.0, - "completions/min_terminated_length": 237.0, - "entropy": 0.1257781982421875, - "epoch": 1.3102366695181067, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03968433290719986, - "kl": 0.0065593719482421875, - "learning_rate": 6.729320366825785e-06, - "loss": 0.004, - "num_tokens": 58954545.0, - "reward": 1.1316603422164917, - "reward_std": 0.3676164150238037, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1316603422164917, - "rewards/cross_solution_unittest_reward_v2/std": 0.539814293384552, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 143 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 936.0, - "completions/mean_length": 522.046875, - "completions/mean_terminated_length": 506.0157165527344, - "completions/min_length": 231.0, - "completions/min_terminated_length": 231.0, - "entropy": 0.1286468505859375, - "epoch": 1.3193612774451098, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03893548995256424, - "kl": 0.006595611572265625, - "learning_rate": 6.579798566743314e-06, - "loss": 0.0302, - "num_tokens": 59365899.0, - "reward": 1.073938012123108, - "reward_std": 0.3488571345806122, - "rewards/cross_solution_unittest_reward_v2/mean": 1.073938012123108, - "rewards/cross_solution_unittest_reward_v2/std": 0.6543789505958557, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 144 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1035.0, - "completions/mean_length": 553.8984375, - "completions/mean_terminated_length": 545.9869384765625, - "completions/min_length": 288.0, - "completions/min_terminated_length": 288.0, - "entropy": 0.1171875, - "epoch": 1.3284858853721129, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.050721071660518646, - "kl": 0.006363391876220703, - "learning_rate": 6.431137784081283e-06, - "loss": 0.0089, - "num_tokens": 59840820.0, - "reward": 1.2019460201263428, - "reward_std": 0.40138086676597595, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2019459009170532, - "rewards/cross_solution_unittest_reward_v2/std": 0.554269015789032, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 145 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.953125, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1370.0, - "completions/mean_length": 536.9088745117188, - "completions/mean_terminated_length": 512.916015625, - "completions/min_length": 237.0, - "completions/min_terminated_length": 237.0, - "entropy": 0.12969970703125, - "epoch": 1.337610493299116, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03871598094701767, - "kl": 0.007752418518066406, - "learning_rate": 6.283375443396726e-06, - "loss": 0.0124, - "num_tokens": 60266297.0, - "reward": 1.1163465976715088, - "reward_std": 0.35231828689575195, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1163464784622192, - "rewards/cross_solution_unittest_reward_v2/std": 0.5736133456230164, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 146 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1058.0, - "completions/mean_length": 527.03125, - "completions/mean_terminated_length": 519.0496215820312, - "completions/min_length": 250.0, - "completions/min_terminated_length": 250.0, - "entropy": 0.135711669921875, - "epoch": 1.3467351012261193, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04133346304297447, - "kl": 0.00714874267578125, - "learning_rate": 6.136548743068713e-06, - "loss": -0.0019, - "num_tokens": 60694413.0, - "reward": 0.9952497482299805, - "reward_std": 0.3892851769924164, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9952497482299805, - "rewards/cross_solution_unittest_reward_v2/std": 0.6389726400375366, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 147 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1131.0, - "completions/max_terminated_length": 1131.0, - "completions/mean_length": 460.9739685058594, - "completions/mean_terminated_length": 460.9739685058594, - "completions/min_length": 215.0, - "completions/min_terminated_length": 215.0, - "entropy": 0.13299560546875, - "epoch": 1.3558597091531224, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.05010734125971794, - "kl": 0.008180618286132812, - "learning_rate": 5.990694645933866e-06, - "loss": -0.0191, - "num_tokens": 61088299.0, - "reward": 1.0312671661376953, - "reward_std": 0.3751637637615204, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0312670469284058, - "rewards/cross_solution_unittest_reward_v2/std": 0.5460102558135986, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 148 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.953125, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 2074.0, - "completions/mean_length": 539.8073120117188, - "completions/mean_terminated_length": 515.8372802734375, - "completions/min_length": 220.0, - "completions/min_terminated_length": 220.0, - "entropy": 0.115234375, - "epoch": 1.3649843170801255, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.034361910074949265, - "kl": 0.005905628204345703, - "learning_rate": 5.845849869981137e-06, - "loss": 0.0006, - "num_tokens": 61467513.0, - "reward": 1.2288246154785156, - "reward_std": 0.31952497363090515, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2288246154785156, - "rewards/cross_solution_unittest_reward_v2/std": 0.46175429224967957, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 149 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 851.0, - "completions/mean_length": 481.8359375, - "completions/mean_terminated_length": 465.59423828125, - "completions/min_length": 244.0, - "completions/min_terminated_length": 244.0, - "entropy": 0.1479034423828125, - "epoch": 1.3741089250071286, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.05979859456419945, - "kl": 0.006015777587890625, - "learning_rate": 5.702050879108284e-06, - "loss": 0.0161, - "num_tokens": 61867242.0, - "reward": 1.0118658542633057, - "reward_std": 0.3811448812484741, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0118657350540161, - "rewards/cross_solution_unittest_reward_v2/std": 0.6259569525718689, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 150 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 873.0, - "completions/max_terminated_length": 873.0, - "completions/mean_length": 481.1875, - "completions/mean_terminated_length": 481.1875, - "completions/min_length": 198.0, - "completions/min_terminated_length": 198.0, - "entropy": 0.1180419921875, - "epoch": 1.3832335329341316, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04232887923717499, - "kl": 0.007142066955566406, - "learning_rate": 5.559333873942259e-06, - "loss": 0.0, - "num_tokens": 62266914.0, - "reward": 1.0256938934326172, - "reward_std": 0.3604298532009125, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0256937742233276, - "rewards/cross_solution_unittest_reward_v2/std": 0.5200158357620239, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 151 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 975.0, - "completions/mean_length": 527.4713745117188, - "completions/mean_terminated_length": 519.4908447265625, - "completions/min_length": 313.0, - "completions/min_terminated_length": 313.0, - "entropy": 0.1217803955078125, - "epoch": 1.392358140861135, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04194113612174988, - "kl": 0.006623268127441406, - "learning_rate": 5.417734782725896e-06, - "loss": 0.0123, - "num_tokens": 62707855.0, - "reward": 0.6789031028747559, - "reward_std": 0.3820572793483734, - "rewards/cross_solution_unittest_reward_v2/mean": 0.6789031028747559, - "rewards/cross_solution_unittest_reward_v2/std": 0.5969597697257996, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 152 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 956.0, - "completions/max_terminated_length": 956.0, - "completions/mean_length": 486.296875, - "completions/mean_terminated_length": 486.296875, - "completions/min_length": 237.0, - "completions/min_terminated_length": 237.0, - "entropy": 0.111572265625, - "epoch": 1.401482748788138, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.041642773896455765, - "kl": 0.006028175354003906, - "learning_rate": 5.277289252273175e-06, - "loss": -0.0096, - "num_tokens": 63095209.0, - "reward": 1.2809089422225952, - "reward_std": 0.3257633447647095, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2809089422225952, - "rewards/cross_solution_unittest_reward_v2/std": 0.4487822353839874, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 153 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 796.0, - "completions/mean_length": 478.6171875, - "completions/mean_terminated_length": 462.358642578125, - "completions/min_length": 222.0, - "completions/min_terminated_length": 222.0, - "entropy": 0.1319122314453125, - "epoch": 1.4106073567151411, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03919358551502228, - "kl": 0.007901191711425781, - "learning_rate": 5.138032638995315e-06, - "loss": 0.0072, - "num_tokens": 63502470.0, - "reward": 1.4154298305511475, - "reward_std": 0.3178824186325073, - "rewards/cross_solution_unittest_reward_v2/mean": 1.415429711341858, - "rewards/cross_solution_unittest_reward_v2/std": 0.5328439474105835, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 154 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.953125, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 884.0, - "completions/mean_length": 508.84375, - "completions/mean_terminated_length": 484.6299133300781, - "completions/min_length": 255.0, - "completions/min_terminated_length": 255.0, - "entropy": 0.1232452392578125, - "epoch": 1.4197319646421442, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04145441949367523, - "kl": 0.006421089172363281, - "learning_rate": 5.000000000000003e-06, - "loss": 0.001, - "num_tokens": 63855346.0, - "reward": 1.2272818088531494, - "reward_std": 0.3207608759403229, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2272816896438599, - "rewards/cross_solution_unittest_reward_v2/std": 0.5835732817649841, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 155 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1027.0, - "completions/max_terminated_length": 1027.0, - "completions/mean_length": 498.39324951171875, - "completions/mean_terminated_length": 498.39324951171875, - "completions/min_length": 269.0, - "completions/min_terminated_length": 269.0, - "entropy": 0.1209564208984375, - "epoch": 1.4288565725691473, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.05915457382798195, - "kl": 0.009290695190429688, - "learning_rate": 4.863226084265939e-06, - "loss": 0.0013, - "num_tokens": 64238729.0, - "reward": 1.0885870456695557, - "reward_std": 0.43416768312454224, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0885871648788452, - "rewards/cross_solution_unittest_reward_v2/std": 0.6135940551757812, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 156 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1044.0, - "completions/mean_length": 497.2239685058594, - "completions/mean_terminated_length": 489.16448974609375, - "completions/min_length": 292.0, - "completions/min_terminated_length": 292.0, - "entropy": 0.1247711181640625, - "epoch": 1.4379811804961506, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03702615946531296, - "kl": 0.006881237030029297, - "learning_rate": 4.727745323894976e-06, - "loss": 0.0034, - "num_tokens": 64650415.0, - "reward": 0.9673380851745605, - "reward_std": 0.4014953076839447, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9673380851745605, - "rewards/cross_solution_unittest_reward_v2/std": 0.6115707159042358, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 157 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 864.0, - "completions/mean_length": 472.0078125, - "completions/mean_terminated_length": 455.71466064453125, - "completions/min_length": 225.0, - "completions/min_terminated_length": 225.0, - "entropy": 0.1342926025390625, - "epoch": 1.4471057884231537, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04624725133180618, - "kl": 0.007852554321289062, - "learning_rate": 4.593591825444028e-06, - "loss": 0.0034, - "num_tokens": 65017242.0, - "reward": 0.9291572570800781, - "reward_std": 0.36605995893478394, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9291572570800781, - "rewards/cross_solution_unittest_reward_v2/std": 0.5336747169494629, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 158 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.90625, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 3521.0, - "completions/mean_length": 566.4661865234375, - "completions/mean_terminated_length": 518.5687866210938, - "completions/min_length": 285.0, - "completions/min_terminated_length": 285.0, - "entropy": 0.126190185546875, - "epoch": 1.4562303963501568, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04579116031527519, - "kl": 0.007121562957763672, - "learning_rate": 4.460799361338898e-06, - "loss": 0.0354, - "num_tokens": 65438397.0, - "reward": 0.986846923828125, - "reward_std": 0.443007230758667, - "rewards/cross_solution_unittest_reward_v2/mean": 0.986846923828125, - "rewards/cross_solution_unittest_reward_v2/std": 0.5646238923072815, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 159 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 922.0, - "completions/mean_length": 503.3515625, - "completions/mean_terminated_length": 487.2225036621094, - "completions/min_length": 276.0, - "completions/min_terminated_length": 276.0, - "entropy": 0.1254119873046875, - "epoch": 1.46535500427716, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.039540164172649384, - "kl": 0.006619453430175781, - "learning_rate": 4.3294013613722944e-06, - "loss": -0.0137, - "num_tokens": 65858772.0, - "reward": 1.0786012411117554, - "reward_std": 0.30854928493499756, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0786011219024658, - "rewards/cross_solution_unittest_reward_v2/std": 0.4968026280403137, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 160 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 889.0, - "completions/max_terminated_length": 889.0, - "completions/mean_length": 443.3099060058594, - "completions/mean_terminated_length": 443.3099060058594, - "completions/min_length": 197.0, - "completions/min_terminated_length": 197.0, - "entropy": 0.135223388671875, - "epoch": 1.474479612204163, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.044837966561317444, - "kl": 0.006756782531738281, - "learning_rate": 4.19943090428802e-06, - "loss": 0.0042, - "num_tokens": 66207587.0, - "reward": 1.4599549770355225, - "reward_std": 0.28187161684036255, - "rewards/cross_solution_unittest_reward_v2/mean": 1.459955096244812, - "rewards/cross_solution_unittest_reward_v2/std": 0.5621427893638611, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 161 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 764.0, - "completions/max_terminated_length": 764.0, - "completions/mean_length": 466.17449951171875, - "completions/mean_terminated_length": 466.17449951171875, - "completions/min_length": 262.0, - "completions/min_terminated_length": 262.0, - "entropy": 0.13555908203125, - "epoch": 1.4836042201311663, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.04105601832270622, - "kl": 0.0064697265625, - "learning_rate": 4.070920709453597e-06, - "loss": 0.0061, - "num_tokens": 66596334.0, - "reward": 1.1189442873001099, - "reward_std": 0.35300740599632263, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1189442873001099, - "rewards/cross_solution_unittest_reward_v2/std": 0.6115458011627197, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 162 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1850.0, - "completions/mean_length": 502.2239685058594, - "completions/mean_terminated_length": 494.17755126953125, - "completions/min_length": 288.0, - "completions/min_terminated_length": 288.0, - "entropy": 0.1320953369140625, - "epoch": 1.4927288280581694, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.044365011155605316, - "kl": 0.006922721862792969, - "learning_rate": 3.943903128623336e-06, - "loss": 0.0056, - "num_tokens": 66999516.0, - "reward": 1.015927791595459, - "reward_std": 0.36806589365005493, - "rewards/cross_solution_unittest_reward_v2/mean": 1.015927791595459, - "rewards/cross_solution_unittest_reward_v2/std": 0.6008942723274231, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 163 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1133.0, - "completions/mean_length": 486.93231201171875, - "completions/mean_terminated_length": 478.845947265625, - "completions/min_length": 185.0, - "completions/min_terminated_length": 185.0, - "entropy": 0.131744384765625, - "epoch": 1.5018534359851725, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.050204940140247345, - "kl": 0.007056713104248047, - "learning_rate": 3.818410137793947e-06, - "loss": 0.0023, - "num_tokens": 67364770.0, - "reward": 0.8856601715087891, - "reward_std": 0.34818410873413086, - "rewards/cross_solution_unittest_reward_v2/mean": 0.8856601715087891, - "rewards/cross_solution_unittest_reward_v2/std": 0.6269282698631287, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 164 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1019.0, - "completions/mean_length": 466.0390625, - "completions/mean_terminated_length": 457.898193359375, - "completions/min_length": 240.0, - "completions/min_terminated_length": 240.0, - "entropy": 0.1310577392578125, - "epoch": 1.5109780439121756, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.042199015617370605, - "kl": 0.006415367126464844, - "learning_rate": 3.6944733291547784e-06, - "loss": 0.0143, - "num_tokens": 67765505.0, - "reward": 1.0736351013183594, - "reward_std": 0.41692399978637695, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0736351013183594, - "rewards/cross_solution_unittest_reward_v2/std": 0.5655834078788757, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 165 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1098.0, - "completions/max_terminated_length": 1098.0, - "completions/mean_length": 493.3411560058594, - "completions/mean_terminated_length": 493.3411560058594, - "completions/min_length": 245.0, - "completions/min_terminated_length": 245.0, - "entropy": 0.1184234619140625, - "epoch": 1.5201026518391787, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.03830646723508835, - "kl": 0.005974292755126953, - "learning_rate": 3.5721239031346067e-06, - "loss": -0.0023, - "num_tokens": 68188724.0, - "reward": 1.4808048009872437, - "reward_std": 0.28709638118743896, - "rewards/cross_solution_unittest_reward_v2/mean": 1.4808048009872437, - "rewards/cross_solution_unittest_reward_v2/std": 0.47111669182777405, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 166 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1005.0, - "completions/max_terminated_length": 1005.0, - "completions/mean_length": 488.33856201171875, - "completions/mean_terminated_length": 488.33856201171875, - "completions/min_length": 193.0, - "completions/min_terminated_length": 193.0, - "entropy": 0.1264190673828125, - "epoch": 1.529227259766182, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03988939896225929, - "kl": 0.0061855316162109375, - "learning_rate": 3.4513926605471504e-06, - "loss": 0.0, - "num_tokens": 68590750.0, - "reward": 1.0762261152267456, - "reward_std": 0.3941044807434082, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0762261152267456, - "rewards/cross_solution_unittest_reward_v2/std": 0.5114349722862244, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 167 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1224.0, - "completions/max_terminated_length": 1224.0, - "completions/mean_length": 489.1692810058594, - "completions/mean_terminated_length": 489.1692810058594, - "completions/min_length": 188.0, - "completions/min_terminated_length": 188.0, - "entropy": 0.1453399658203125, - "epoch": 1.538351867693185, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.038756903260946274, - "kl": 0.006499290466308594, - "learning_rate": 3.3323099948370853e-06, - "loss": -0.0086, - "num_tokens": 68998335.0, - "reward": 1.1381275653839111, - "reward_std": 0.27171847224235535, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1381274461746216, - "rewards/cross_solution_unittest_reward_v2/std": 0.5886113047599792, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 168 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.953125, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1124.0, - "completions/mean_length": 514.1198120117188, - "completions/mean_terminated_length": 489.947509765625, - "completions/min_length": 226.0, - "completions/min_terminated_length": 226.0, - "entropy": 0.1281280517578125, - "epoch": 1.5474764756201882, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03642740473151207, - "kl": 0.007023334503173828, - "learning_rate": 3.2149058844286796e-06, - "loss": -0.0029, - "num_tokens": 69386893.0, - "reward": 1.049883246421814, - "reward_std": 0.434080570936203, - "rewards/cross_solution_unittest_reward_v2/mean": 1.049883246421814, - "rewards/cross_solution_unittest_reward_v2/std": 0.5577353239059448, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 169 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 902.0, - "completions/mean_length": 495.0442810058594, - "completions/mean_terminated_length": 486.9791259765625, - "completions/min_length": 185.0, - "completions/min_terminated_length": 185.0, - "entropy": 0.128509521484375, - "epoch": 1.5566010835471913, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03712453693151474, - "kl": 0.005021095275878906, - "learning_rate": 3.099209885178882e-06, - "loss": 0.0005, - "num_tokens": 69782118.0, - "reward": 1.0256097316741943, - "reward_std": 0.3787035346031189, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0256096124649048, - "rewards/cross_solution_unittest_reward_v2/std": 0.6416430473327637, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 170 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 802.0, - "completions/mean_length": 472.3828125, - "completions/mean_terminated_length": 456.09161376953125, - "completions/min_length": 150.0, - "completions/min_terminated_length": 150.0, - "entropy": 0.1376190185546875, - "epoch": 1.5657256914741944, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.03793985769152641, - "kl": 0.006984710693359375, - "learning_rate": 2.9852511229367862e-06, - "loss": 0.0206, - "num_tokens": 70161569.0, - "reward": 1.0312031507492065, - "reward_std": 0.3098965883255005, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0312031507492065, - "rewards/cross_solution_unittest_reward_v2/std": 0.6053119897842407, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 171 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1198.0, - "completions/max_terminated_length": 1198.0, - "completions/mean_length": 526.2552490234375, - "completions/mean_terminated_length": 526.2552490234375, - "completions/min_length": 252.0, - "completions/min_terminated_length": 252.0, - "entropy": 0.11407470703125, - "epoch": 1.5748502994011977, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.0374041423201561, - "kl": 0.005502223968505859, - "learning_rate": 2.8730582862113743e-06, - "loss": -0.0045, - "num_tokens": 70572715.0, - "reward": 1.2637739181518555, - "reward_std": 0.30983880162239075, - "rewards/cross_solution_unittest_reward_v2/mean": 1.263774037361145, - "rewards/cross_solution_unittest_reward_v2/std": 0.5244058966636658, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 172 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 919.0, - "completions/max_terminated_length": 919.0, - "completions/mean_length": 492.65106201171875, - "completions/mean_terminated_length": 492.65106201171875, - "completions/min_length": 251.0, - "completions/min_terminated_length": 251.0, - "entropy": 0.1183929443359375, - "epoch": 1.5839749073282008, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.03538573160767555, - "kl": 0.005868434906005859, - "learning_rate": 2.7626596189492983e-06, - "loss": -0.0022, - "num_tokens": 70979941.0, - "reward": 1.339817762374878, - "reward_std": 0.2546539902687073, - "rewards/cross_solution_unittest_reward_v2/mean": 1.3398176431655884, - "rewards/cross_solution_unittest_reward_v2/std": 0.4908412992954254, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 173 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1269.0, - "completions/mean_length": 459.234375, - "completions/mean_terminated_length": 442.8743591308594, - "completions/min_length": 223.0, - "completions/min_terminated_length": 223.0, - "entropy": 0.135650634765625, - "epoch": 1.5930995152552039, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03862559050321579, - "kl": 0.007285118103027344, - "learning_rate": 2.6540829134246683e-06, - "loss": -0.0041, - "num_tokens": 71317671.0, - "reward": 1.2179704904556274, - "reward_std": 0.3110223710536957, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2179704904556274, - "rewards/cross_solution_unittest_reward_v2/std": 0.6325693726539612, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 174 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1057.0, - "completions/max_terminated_length": 1057.0, - "completions/mean_length": 469.609375, - "completions/mean_terminated_length": 469.609375, - "completions/min_length": 247.0, - "completions/min_terminated_length": 247.0, - "entropy": 0.1310577392578125, - "epoch": 1.602224123182207, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.05514206364750862, - "kl": 0.007549762725830078, - "learning_rate": 2.5473555032424534e-06, - "loss": -0.0109, - "num_tokens": 71684737.0, - "reward": 1.3404430150985718, - "reward_std": 0.27716976404190063, - "rewards/cross_solution_unittest_reward_v2/mean": 1.3404430150985718, - "rewards/cross_solution_unittest_reward_v2/std": 0.5657073259353638, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 175 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1022.0, - "completions/mean_length": 463.390625, - "completions/mean_terminated_length": 455.2428283691406, - "completions/min_length": 216.0, - "completions/min_terminated_length": 216.0, - "entropy": 0.130462646484375, - "epoch": 1.61134873110921, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04906470701098442, - "kl": 0.00655364990234375, - "learning_rate": 2.4425042564574186e-06, - "loss": -0.012, - "num_tokens": 72073503.0, - "reward": 1.0603030920028687, - "reward_std": 0.30689382553100586, - "rewards/cross_solution_unittest_reward_v2/mean": 1.060302972793579, - "rewards/cross_solution_unittest_reward_v2/std": 0.6134819388389587, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 176 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 930.0, - "completions/max_terminated_length": 930.0, - "completions/mean_length": 510.3203125, - "completions/mean_terminated_length": 510.3203125, - "completions/min_length": 268.0, - "completions/min_terminated_length": 268.0, - "entropy": 0.1191864013671875, - "epoch": 1.6204733390362134, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.039267756044864655, - "kl": 0.0056781768798828125, - "learning_rate": 2.339555568810221e-06, - "loss": 0.0073, - "num_tokens": 72509242.0, - "reward": 1.334220290184021, - "reward_std": 0.36035117506980896, - "rewards/cross_solution_unittest_reward_v2/mean": 1.334220290184021, - "rewards/cross_solution_unittest_reward_v2/std": 0.6142004728317261, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 177 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1229.0, - "completions/mean_length": 504.0859375, - "completions/mean_terminated_length": 496.0444030761719, - "completions/min_length": 260.0, - "completions/min_terminated_length": 260.0, - "entropy": 0.1092681884765625, - "epoch": 1.6295979469632165, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.041319265961647034, - "kl": 0.005724430084228516, - "learning_rate": 2.2385353570824308e-06, - "loss": -0.0012, - "num_tokens": 72954091.0, - "reward": 0.9998946189880371, - "reward_std": 0.32818570733070374, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9998946189880371, - "rewards/cross_solution_unittest_reward_v2/std": 0.6151955723762512, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 178 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1310.0, - "completions/max_terminated_length": 1310.0, - "completions/mean_length": 526.6666870117188, - "completions/mean_terminated_length": 526.6666870117188, - "completions/min_length": 273.0, - "completions/min_terminated_length": 273.0, - "entropy": 0.1173553466796875, - "epoch": 1.6387225548902196, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03957797959446907, - "kl": 0.005245208740234375, - "learning_rate": 2.1394690525721275e-06, - "loss": 0.0046, - "num_tokens": 73405731.0, - "reward": 1.1459977626800537, - "reward_std": 0.2727368474006653, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1459976434707642, - "rewards/cross_solution_unittest_reward_v2/std": 0.6138551235198975, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 179 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1501.0, - "completions/mean_length": 533.0260620117188, - "completions/mean_terminated_length": 517.0523681640625, - "completions/min_length": 258.0, - "completions/min_terminated_length": 258.0, - "entropy": 0.120574951171875, - "epoch": 1.6478471628172227, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.041543569415807724, - "kl": 0.005976676940917969, - "learning_rate": 2.0423815946916783e-06, - "loss": 0.0081, - "num_tokens": 73826037.0, - "reward": 1.1596499681472778, - "reward_std": 0.25536206364631653, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1596498489379883, - "rewards/cross_solution_unittest_reward_v2/std": 0.6891902685165405, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 180 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 948.0, - "completions/max_terminated_length": 948.0, - "completions/mean_length": 509.28387451171875, - "completions/mean_terminated_length": 509.28387451171875, - "completions/min_length": 195.0, - "completions/min_terminated_length": 195.0, - "entropy": 0.12506103515625, - "epoch": 1.6569717707442257, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04603911563754082, - "kl": 0.005138397216796875, - "learning_rate": 1.947297424689414e-06, - "loss": 0.0013, - "num_tokens": 74225978.0, - "reward": 1.0025129318237305, - "reward_std": 0.4077932834625244, - "rewards/cross_solution_unittest_reward_v2/mean": 1.002512812614441, - "rewards/cross_solution_unittest_reward_v2/std": 0.6445339918136597, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 181 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1215.0, - "completions/mean_length": 526.1588745117188, - "completions/mean_terminated_length": 510.14923095703125, - "completions/min_length": 267.0, - "completions/min_terminated_length": 267.0, - "entropy": 0.119476318359375, - "epoch": 1.666096378671229, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.03396144509315491, - "kl": 0.00534820556640625, - "learning_rate": 1.854240479496643e-06, - "loss": 0.02, - "num_tokens": 74661783.0, - "reward": 1.0279009342193604, - "reward_std": 0.343097060918808, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0279008150100708, - "rewards/cross_solution_unittest_reward_v2/std": 0.622691810131073, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 182 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1332.0, - "completions/max_terminated_length": 1332.0, - "completions/mean_length": 498.1692810058594, - "completions/mean_terminated_length": 498.1692810058594, - "completions/min_length": 237.0, - "completions/min_terminated_length": 237.0, - "entropy": 0.123748779296875, - "epoch": 1.6752209865982322, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04850158095359802, - "kl": 0.007281303405761719, - "learning_rate": 1.7632341857016733e-06, - "loss": -0.007, - "num_tokens": 75054392.0, - "reward": 0.9975048899650574, - "reward_std": 0.26312822103500366, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9975049495697021, - "rewards/cross_solution_unittest_reward_v2/std": 0.5667267441749573, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 183 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 967.0, - "completions/mean_length": 512.0833740234375, - "completions/mean_terminated_length": 504.06268310546875, - "completions/min_length": 271.0, - "completions/min_terminated_length": 271.0, - "entropy": 0.1220550537109375, - "epoch": 1.6843455945252352, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.040891263633966446, - "kl": 0.005091190338134766, - "learning_rate": 1.6743014536522872e-06, - "loss": 0.0116, - "num_tokens": 75473584.0, - "reward": 0.9722011089324951, - "reward_std": 0.3683922290802002, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9722011089324951, - "rewards/cross_solution_unittest_reward_v2/std": 0.5778730511665344, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 184 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 934.0, - "completions/mean_length": 463.8645935058594, - "completions/mean_terminated_length": 455.718017578125, - "completions/min_length": 231.0, - "completions/min_terminated_length": 231.0, - "entropy": 0.135772705078125, - "epoch": 1.6934702024522383, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03959479182958603, - "kl": 0.006596088409423828, - "learning_rate": 1.587464671688187e-06, - "loss": 0.0091, - "num_tokens": 75825860.0, - "reward": 1.3711402416229248, - "reward_std": 0.3146709203720093, - "rewards/cross_solution_unittest_reward_v2/mean": 1.3711403608322144, - "rewards/cross_solution_unittest_reward_v2/std": 0.5123069882392883, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 185 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1215.0, - "completions/mean_length": 517.3046875, - "completions/mean_terminated_length": 509.29766845703125, - "completions/min_length": 244.0, - "completions/min_terminated_length": 244.0, - "entropy": 0.120330810546875, - "epoch": 1.7025948103792414, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04447691887617111, - "kl": 0.006220817565917969, - "learning_rate": 1.5027457005048573e-06, - "loss": 0.0056, - "num_tokens": 76252713.0, - "reward": 0.9647834300994873, - "reward_std": 0.377196729183197, - "rewards/cross_solution_unittest_reward_v2/mean": 0.9647834300994873, - "rewards/cross_solution_unittest_reward_v2/std": 0.5407822132110596, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 186 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.9375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 2752.0, - "completions/mean_length": 503.0833435058594, - "completions/mean_terminated_length": 470.65264892578125, - "completions/min_length": 264.0, - "completions/min_terminated_length": 264.0, - "entropy": 0.1254730224609375, - "epoch": 1.7117194183062447, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.043862879276275635, - "kl": 0.006692409515380859, - "learning_rate": 1.4201658676502294e-06, - "loss": 0.0135, - "num_tokens": 76623081.0, - "reward": 1.1441748142242432, - "reward_std": 0.36825141310691833, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1441746950149536, - "rewards/cross_solution_unittest_reward_v2/std": 0.6024966239929199, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 187 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1332.0, - "completions/max_terminated_length": 1332.0, - "completions/mean_length": 496.58856201171875, - "completions/mean_terminated_length": 496.58856201171875, - "completions/min_length": 248.0, - "completions/min_terminated_length": 248.0, - "entropy": 0.1305389404296875, - "epoch": 1.7208440262332478, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.0364561527967453, - "kl": 0.006022930145263672, - "learning_rate": 1.339745962155613e-06, - "loss": 0.013, - "num_tokens": 77010211.0, - "reward": 1.2899563312530518, - "reward_std": 0.2495330274105072, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2899564504623413, - "rewards/cross_solution_unittest_reward_v2/std": 0.636502742767334, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 188 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.96875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1540.0, - "completions/mean_length": 520.40625, - "completions/mean_terminated_length": 504.3664855957031, - "completions/min_length": 234.0, - "completions/min_terminated_length": 234.0, - "entropy": 0.1340179443359375, - "epoch": 1.729968634160251, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.05484146252274513, - "kl": 0.006550788879394531, - "learning_rate": 1.2615062293021508e-06, - "loss": 0.0187, - "num_tokens": 77373911.0, - "reward": 1.157588005065918, - "reward_std": 0.37545454502105713, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1575878858566284, - "rewards/cross_solution_unittest_reward_v2/std": 0.5925281047821045, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 189 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1011.0, - "completions/mean_length": 503.7630310058594, - "completions/mean_terminated_length": 495.72064208984375, - "completions/min_length": 230.0, - "completions/min_terminated_length": 230.0, - "entropy": 0.1233673095703125, - "epoch": 1.739093242087254, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03927004709839821, - "kl": 0.004923343658447266, - "learning_rate": 1.1854663655241804e-06, - "loss": 0.0152, - "num_tokens": 77780244.0, - "reward": 1.1829155683517456, - "reward_std": 0.33043980598449707, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1829155683517456, - "rewards/cross_solution_unittest_reward_v2/std": 0.5110778212547302, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 190 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 884.0, - "completions/max_terminated_length": 884.0, - "completions/mean_length": 497.3099060058594, - "completions/mean_terminated_length": 497.3099060058594, - "completions/min_length": 222.0, - "completions/min_terminated_length": 222.0, - "entropy": 0.1133575439453125, - "epoch": 1.7482178500142571, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.03486226126551628, - "kl": 0.005496025085449219, - "learning_rate": 1.1116455134507665e-06, - "loss": 0.0069, - "num_tokens": 78205875.0, - "reward": 1.1240648031234741, - "reward_std": 0.38483256101608276, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1240648031234741, - "rewards/cross_solution_unittest_reward_v2/std": 0.5771324038505554, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 191 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1111.0, - "completions/mean_length": 481.51824951171875, - "completions/mean_terminated_length": 473.4177551269531, - "completions/min_length": 254.0, - "completions/min_terminated_length": 254.0, - "entropy": 0.145050048828125, - "epoch": 1.7573424579412604, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03699757531285286, - "kl": 0.0060214996337890625, - "learning_rate": 1.0400622570866426e-06, - "loss": 0.0052, - "num_tokens": 78614530.0, - "reward": 1.3582351207733154, - "reward_std": 0.4048115909099579, - "rewards/cross_solution_unittest_reward_v2/mean": 1.3582350015640259, - "rewards/cross_solution_unittest_reward_v2/std": 0.5887612104415894, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 192 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1149.0, - "completions/mean_length": 519.5, - "completions/mean_terminated_length": 511.4986877441406, - "completions/min_length": 224.0, - "completions/min_terminated_length": 224.0, - "entropy": 0.118682861328125, - "epoch": 1.7664670658682635, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03480589762330055, - "kl": 0.005451202392578125, - "learning_rate": 9.707346171337895e-07, - "loss": 0.0049, - "num_tokens": 79036634.0, - "reward": 1.1587016582489014, - "reward_std": 0.30175238847732544, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1587015390396118, - "rewards/cross_solution_unittest_reward_v2/std": 0.5676968693733215, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 193 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1817.0, - "completions/max_terminated_length": 1817.0, - "completions/mean_length": 537.3932495117188, - "completions/mean_terminated_length": 537.3932495117188, - "completions/min_length": 206.0, - "completions/min_terminated_length": 206.0, - "entropy": 0.10357666015625, - "epoch": 1.7755916737952666, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.03492164984345436, - "kl": 0.00434112548828125, - "learning_rate": 9.036800464548157e-07, - "loss": -0.0005, - "num_tokens": 79481225.0, - "reward": 1.1358517408370972, - "reward_std": 0.19027307629585266, - "rewards/cross_solution_unittest_reward_v2/mean": 1.1358517408370972, - "rewards/cross_solution_unittest_reward_v2/std": 0.6591117978096008, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 194 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.9375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 855.0, - "completions/mean_length": 514.4401245117188, - "completions/mean_terminated_length": 482.12896728515625, - "completions/min_length": 236.0, - "completions/min_terminated_length": 236.0, - "entropy": 0.1154937744140625, - "epoch": 1.7847162817222697, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.05267718434333801, - "kl": 0.005501747131347656, - "learning_rate": 8.389154256793042e-07, - "loss": 0.0223, - "num_tokens": 79868074.0, - "reward": 1.3394169807434082, - "reward_std": 0.4405336380004883, - "rewards/cross_solution_unittest_reward_v2/mean": 1.3394169807434082, - "rewards/cross_solution_unittest_reward_v2/std": 0.5766070485115051, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 195 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.9375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 995.0, - "completions/mean_length": 555.0130615234375, - "completions/mean_terminated_length": 523.1289672851562, - "completions/min_length": 290.0, - "completions/min_terminated_length": 290.0, - "entropy": 0.1145477294921875, - "epoch": 1.7938408896492728, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.036951858550310135, - "kl": 0.006405830383300781, - "learning_rate": 7.764570589541876e-07, - "loss": 0.0153, - "num_tokens": 80332583.0, - "reward": 1.0603303909301758, - "reward_std": 0.3560698926448822, - "rewards/cross_solution_unittest_reward_v2/mean": 1.0603302717208862, - "rewards/cross_solution_unittest_reward_v2/std": 0.7043731808662415, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 196 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 1020.0, - "completions/max_terminated_length": 1020.0, - "completions/mean_length": 560.3802490234375, - "completions/mean_terminated_length": 560.3802490234375, - "completions/min_length": 243.0, - "completions/min_terminated_length": 243.0, - "entropy": 0.1178436279296875, - "epoch": 1.802965497576276, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.04354370757937431, - "kl": 0.006405830383300781, - "learning_rate": 7.163206698392744e-07, - "loss": 0.0127, - "num_tokens": 80807537.0, - "reward": 0.999901294708252, - "reward_std": 0.3926628828048706, - "rewards/cross_solution_unittest_reward_v2/mean": 0.999901294708252, - "rewards/cross_solution_unittest_reward_v2/std": 0.4642387628555298, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 197 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.984375, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 1012.0, - "completions/mean_length": 514.2682495117188, - "completions/mean_terminated_length": 506.2532653808594, - "completions/min_length": 239.0, - "completions/min_terminated_length": 239.0, - "entropy": 0.1213836669921875, - "epoch": 1.8120901055032792, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.03938065469264984, - "kl": 0.005326747894287109, - "learning_rate": 6.585213973489335e-07, - "loss": 0.0085, - "num_tokens": 81221344.0, - "reward": 1.081814169883728, - "reward_std": 0.3103455901145935, - "rewards/cross_solution_unittest_reward_v2/mean": 1.081814169883728, - "rewards/cross_solution_unittest_reward_v2/std": 0.7211072444915771, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 198 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -5.0, - "completions/max_length": 762.0, - "completions/max_terminated_length": 762.0, - "completions/mean_length": 492.2578125, - "completions/mean_terminated_length": 492.2578125, - "completions/min_length": 273.0, - "completions/min_terminated_length": 273.0, - "entropy": 0.1223297119140625, - "epoch": 1.8212147134302823, - "frac_reward_zero_std": 0.0833333358168602, - "grad_norm": 0.038712866604328156, - "kl": 0.005485057830810547, - "learning_rate": 6.030737921409169e-07, - "loss": -0.0002, - "num_tokens": 81615915.0, - "reward": 1.5546224117279053, - "reward_std": 0.24751900136470795, - "rewards/cross_solution_unittest_reward_v2/mean": 1.5546221733093262, - "rewards/cross_solution_unittest_reward_v2/std": 0.5098509192466736, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 199 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": -4.921875, - "completions/max_length": 3584.0, - "completions/max_terminated_length": 3429.0, - "completions/mean_length": 582.2526245117188, - "completions/mean_terminated_length": 542.6517333984375, - "completions/min_length": 207.0, - "completions/min_terminated_length": 207.0, - "entropy": 0.11956787109375, - "epoch": 1.8303393213572854, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.039214201271533966, - "kl": 0.005737781524658203, - "learning_rate": 5.499918128533155e-07, - "loss": 0.0195, - "num_tokens": 82038884.0, - "reward": 1.2620351314544678, - "reward_std": 0.3703279197216034, - "rewards/cross_solution_unittest_reward_v2/mean": 1.2620350122451782, - "rewards/cross_solution_unittest_reward_v2/std": 0.5252670645713806, - "rewards/format_reward/mean": 0.0, - "rewards/format_reward/std": 0.0, - "step": 200 - }, - { - "epoch": 1.8303393213572854, - "step": 200, + "epoch": 0.9945822640433419, + "step": 109, "total_flos": 0.0, - "train_loss": 0.0, - "train_runtime": 0.1209, - "train_samples_per_second": 10872.756, - "train_steps_per_second": 909.508 + "train_loss": 0.0018782940647274324, + "train_runtime": 42443.7986, + "train_samples_per_second": 0.031, + "train_steps_per_second": 0.003 } ], "logging_steps": 1, "max_steps": 110, - "num_input_tokens_seen": 82038884, + "num_input_tokens_seen": 44271795, "num_train_epochs": 1, "save_steps": 25, "stateful_callbacks": {