diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9840546697038725,
+  "epoch": 0.9945822640433419,
   "eval_steps": 500,
-  "global_step": 27,
+  "global_step": 109,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -16,24 +16,24 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -5.0,
-      "completions/max_length": 1030.0,
-      "completions/max_terminated_length": 1030.0,
-      "completions/mean_length": 520.4375,
-      "completions/mean_terminated_length": 520.4375,
-      "completions/min_length": 259.0,
-      "completions/min_terminated_length": 259.0,
-      "entropy": 0.0713653564453125,
-      "epoch": 0.03644646924829157,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.02890011854469776,
-      "kl": 0.00021544098854064941,
+      "completions/max_length": 1070.0,
+      "completions/max_terminated_length": 1070.0,
+      "completions/mean_length": 533.8359375,
+      "completions/mean_terminated_length": 533.8359375,
+      "completions/min_length": 242.0,
+      "completions/min_terminated_length": 242.0,
+      "entropy": 0.0727691650390625,
+      "epoch": 0.009124607927003137,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.023102419450879097,
+      "kl": 0.00021660327911376953,
       "learning_rate": 0.0,
-      "loss": -0.0031,
-      "num_tokens": 211542.0,
-      "reward": 0.5595489144325256,
-      "reward_std": 0.10752050578594208,
-      "rewards/cross_solution_unittest/mean": 0.5595488548278809,
-      "rewards/cross_solution_unittest/std": 0.18392445147037506,
+      "loss": -0.0135,
+      "num_tokens": 413577.0,
+      "reward": 0.6045712828636169,
+      "reward_std": 0.06758441776037216,
+      "rewards/cross_solution_unittest/mean": 0.6045712828636169,
+      "rewards/cross_solution_unittest/std": 0.09926356375217438,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 1
@@ -44,25 +44,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -4.90625,
-      "completions/max_length": 3584.0,
-      "completions/max_terminated_length": 973.0,
-      "completions/mean_length": 565.9271240234375,
-      "completions/mean_terminated_length": 518.0211181640625,
-      "completions/min_length": 220.0,
-      "completions/min_terminated_length": 220.0,
-      "entropy": 0.0749359130859375,
-      "epoch": 0.07289293849658314,
-      "frac_reward_zero_std": 0.02083333395421505,
-      "grad_norm": 0.02937042899429798,
-      "kl": 0.00021922588348388672,
-      "learning_rate": 6.666666666666667e-06,
-      "loss": -0.0023,
-      "num_tokens": 432880.0,
-      "reward": 0.616872251033783,
-      "reward_std": 0.10939260572195053,
-      "rewards/cross_solution_unittest/mean": 0.6168721914291382,
-      "rewards/cross_solution_unittest/std": 0.19950935244560242,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1010.0,
+      "completions/max_terminated_length": 1010.0,
+      "completions/mean_length": 495.5234375,
+      "completions/mean_terminated_length": 495.5234375,
+      "completions/min_length": 219.0,
+      "completions/min_terminated_length": 219.0,
+      "entropy": 0.07559967041015625,
+      "epoch": 0.018249215854006275,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.028239840641617775,
+      "kl": 0.00024643540382385254,
+      "learning_rate": 1.8181818181818183e-06,
+      "loss": -0.0058,
+      "num_tokens": 824930.0,
+      "reward": 0.5676477551460266,
+      "reward_std": 0.07896548509597778,
+      "rewards/cross_solution_unittest/mean": 0.5676476955413818,
+      "rewards/cross_solution_unittest/std": 0.12314769625663757,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 2
@@ -74,24 +74,24 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -5.0,
-      "completions/max_length": 1322.0,
-      "completions/max_terminated_length": 1322.0,
-      "completions/mean_length": 517.7291870117188,
-      "completions/mean_terminated_length": 517.7291870117188,
-      "completions/min_length": 273.0,
-      "completions/min_terminated_length": 273.0,
-      "entropy": 0.0756378173828125,
-      "epoch": 0.10933940774487472,
+      "completions/max_length": 1136.0,
+      "completions/max_terminated_length": 1136.0,
+      "completions/mean_length": 568.5729370117188,
+      "completions/mean_terminated_length": 568.5729370117188,
+      "completions/min_length": 316.0,
+      "completions/min_terminated_length": 316.0,
+      "entropy": 0.0638427734375,
+      "epoch": 0.02737382378100941,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.03240806981921196,
-      "kl": 0.00023889541625976562,
-      "learning_rate": 1.3333333333333333e-05,
-      "loss": -0.0129,
-      "num_tokens": 645970.0,
-      "reward": 0.5990354418754578,
-      "reward_std": 0.1072663888335228,
-      "rewards/cross_solution_unittest/mean": 0.5990354418754578,
-      "rewards/cross_solution_unittest/std": 0.19944854080677032,
+      "grad_norm": 0.026136187836527824,
+      "kl": 0.00020660459995269775,
+      "learning_rate": 3.6363636363636366e-06,
+      "loss": 0.0073,
+      "num_tokens": 1260498.0,
+      "reward": 0.5541688203811646,
+      "reward_std": 0.05321730300784111,
+      "rewards/cross_solution_unittest/mean": 0.5541687607765198,
+      "rewards/cross_solution_unittest/std": 0.11517918854951859,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 3
@@ -102,25 +102,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -4.96875,
+      "completions/clipped_ratio": -4.984375,
       "completions/max_length": 3584.0,
-      "completions/max_terminated_length": 1919.0,
-      "completions/mean_length": 533.5052490234375,
-      "completions/mean_terminated_length": 517.5340576171875,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
-      "entropy": 0.0742034912109375,
-      "epoch": 0.14578587699316628,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.02899506501853466,
-      "kl": 0.0002148747444152832,
-      "learning_rate": 2e-05,
-      "loss": -0.0112,
-      "num_tokens": 848277.0,
-      "reward": 0.5434086322784424,
-      "reward_std": 0.09839596599340439,
-      "rewards/cross_solution_unittest/mean": 0.5434086322784424,
-      "rewards/cross_solution_unittest/std": 0.21125413477420807,
+      "completions/max_terminated_length": 845.0,
+      "completions/mean_length": 497.7317810058594,
+      "completions/mean_terminated_length": 489.67364501953125,
+      "completions/min_length": 297.0,
+      "completions/min_terminated_length": 297.0,
+      "entropy": 0.0802001953125,
+      "epoch": 0.03649843170801255,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.027541937306523323,
+      "kl": 0.0002409219741821289,
+      "learning_rate": 5.4545454545454545e-06,
+      "loss": -0.0068,
+      "num_tokens": 1696939.0,
+      "reward": 0.5838931798934937,
+      "reward_std": 0.07090283930301666,
+      "rewards/cross_solution_unittest/mean": 0.5838931202888489,
+      "rewards/cross_solution_unittest/std": 0.10168680548667908,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 4
@@ -131,25 +131,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -5.0,
-      "completions/max_length": 929.0,
-      "completions/max_terminated_length": 929.0,
-      "completions/mean_length": 512.15625,
-      "completions/mean_terminated_length": 512.15625,
-      "completions/min_length": 241.0,
-      "completions/min_terminated_length": 241.0,
-      "entropy": 0.074676513671875,
-      "epoch": 0.18223234624145787,
-      "frac_reward_zero_std": 0.0625,
-      "grad_norm": 0.030133171007037163,
-      "kl": 0.0002295374870300293,
-      "learning_rate": 1.9921147013144782e-05,
-      "loss": -0.0018,
-      "num_tokens": 1058649.0,
-      "reward": 0.5704458951950073,
-      "reward_std": 0.10581853240728378,
-      "rewards/cross_solution_unittest/mean": 0.5704458951950073,
-      "rewards/cross_solution_unittest/std": 0.20132118463516235,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 864.0,
+      "completions/mean_length": 490.75,
+      "completions/mean_terminated_length": 482.67364501953125,
+      "completions/min_length": 263.0,
+      "completions/min_terminated_length": 263.0,
+      "entropy": 0.082305908203125,
+      "epoch": 0.04562303963501568,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.027760116383433342,
+      "kl": 0.0002455413341522217,
+      "learning_rate": 7.272727272727273e-06,
+      "loss": -0.0016,
+      "num_tokens": 2080287.0,
+      "reward": 0.6031867861747742,
+      "reward_std": 0.05878433585166931,
+      "rewards/cross_solution_unittest/mean": 0.6031867861747742,
+      "rewards/cross_solution_unittest/std": 0.12897227704524994,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 5
@@ -160,25 +160,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -4.96875,
+      "completions/clipped_ratio": -4.984375,
       "completions/max_length": 3584.0,
-      "completions/max_terminated_length": 1124.0,
-      "completions/mean_length": 533.1354370117188,
-      "completions/mean_terminated_length": 517.1622924804688,
-      "completions/min_length": 265.0,
-      "completions/min_terminated_length": 265.0,
-      "entropy": 0.07135009765625,
-      "epoch": 0.21867881548974943,
-      "frac_reward_zero_std": 0.02083333395421505,
-      "grad_norm": 0.030490361154079437,
-      "kl": 0.00021791458129882812,
-      "learning_rate": 1.9685831611286312e-05,
-      "loss": 0.0,
-      "num_tokens": 1267713.0,
-      "reward": 0.579592227935791,
-      "reward_std": 0.12655234336853027,
-      "rewards/cross_solution_unittest/mean": 0.579592227935791,
-      "rewards/cross_solution_unittest/std": 0.2343016117811203,
+      "completions/max_terminated_length": 1842.0,
+      "completions/mean_length": 544.1614990234375,
+      "completions/mean_terminated_length": 536.2245483398438,
+      "completions/min_length": 290.0,
+      "completions/min_terminated_length": 290.0,
+      "entropy": 0.077362060546875,
+      "epoch": 0.05474764756201882,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.024699553847312927,
+      "kl": 0.00022616982460021973,
+      "learning_rate": 9.090909090909091e-06,
+      "loss": -0.0098,
+      "num_tokens": 2505261.0,
+      "reward": 0.6043218374252319,
+      "reward_std": 0.049580853432416916,
+      "rewards/cross_solution_unittest/mean": 0.6043217778205872,
+      "rewards/cross_solution_unittest/std": 0.08841408789157867,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 6
@@ -190,24 +190,24 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -5.0,
-      "completions/max_length": 880.0,
-      "completions/max_terminated_length": 880.0,
-      "completions/mean_length": 505.27606201171875,
-      "completions/mean_terminated_length": 505.27606201171875,
-      "completions/min_length": 233.0,
-      "completions/min_terminated_length": 233.0,
-      "entropy": 0.07843017578125,
-      "epoch": 0.255125284738041,
+      "completions/max_length": 1419.0,
+      "completions/max_terminated_length": 1419.0,
+      "completions/mean_length": 537.2604370117188,
+      "completions/mean_terminated_length": 537.2604370117188,
+      "completions/min_length": 257.0,
+      "completions/min_terminated_length": 257.0,
+      "entropy": 0.0787200927734375,
+      "epoch": 0.06387225548902195,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.031553227454423904,
-      "kl": 0.00024181604385375977,
-      "learning_rate": 1.9297764858882516e-05,
-      "loss": 0.0249,
-      "num_tokens": 1467406.0,
-      "reward": 0.5725040435791016,
-      "reward_std": 0.10377390682697296,
-      "rewards/cross_solution_unittest/mean": 0.5725040435791016,
-      "rewards/cross_solution_unittest/std": 0.21926818788051605,
+      "grad_norm": 0.02539636939764023,
+      "kl": 0.00023606419563293457,
+      "learning_rate": 1.0909090909090909e-05,
+      "loss": -0.01,
+      "num_tokens": 2952069.0,
+      "reward": 0.597324788570404,
+      "reward_std": 0.07878245413303375,
+      "rewards/cross_solution_unittest/mean": 0.597324788570404,
+      "rewards/cross_solution_unittest/std": 0.11920402199029922,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 7
@@ -218,25 +218,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -4.9375,
-      "completions/max_length": 3584.0,
-      "completions/max_terminated_length": 917.0,
-      "completions/mean_length": 520.34375,
-      "completions/mean_terminated_length": 488.0947570800781,
-      "completions/min_length": 277.0,
-      "completions/min_terminated_length": 277.0,
-      "entropy": 0.0777130126953125,
-      "epoch": 0.29157175398633256,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.030870754271745682,
-      "kl": 0.00022351741790771484,
-      "learning_rate": 1.8763066800438638e-05,
-      "loss": 0.0048,
-      "num_tokens": 1677876.0,
-      "reward": 0.5693823099136353,
-      "reward_std": 0.12342718243598938,
-      "rewards/cross_solution_unittest/mean": 0.5693822503089905,
-      "rewards/cross_solution_unittest/std": 0.22165222465991974,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1060.0,
+      "completions/max_terminated_length": 1060.0,
+      "completions/mean_length": 527.5234375,
+      "completions/mean_terminated_length": 527.5234375,
+      "completions/min_length": 246.0,
+      "completions/min_terminated_length": 246.0,
+      "entropy": 0.074920654296875,
+      "epoch": 0.0729968634160251,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.025767840445041656,
+      "kl": 0.0002295374870300293,
+      "learning_rate": 1.2727272727272728e-05,
+      "loss": -0.0017,
+      "num_tokens": 3409458.0,
+      "reward": 0.546064019203186,
+      "reward_std": 0.06948773562908173,
+      "rewards/cross_solution_unittest/mean": 0.5460640788078308,
+      "rewards/cross_solution_unittest/std": 0.11339057236909866,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 8
@@ -248,24 +248,24 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -5.0,
-      "completions/max_length": 1040.0,
-      "completions/max_terminated_length": 1040.0,
-      "completions/mean_length": 500.7864685058594,
-      "completions/mean_terminated_length": 500.7864685058594,
-      "completions/min_length": 308.0,
-      "completions/min_terminated_length": 308.0,
-      "entropy": 0.07794189453125,
-      "epoch": 0.32801822323462415,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.030629878863692284,
-      "kl": 0.00024515390396118164,
-      "learning_rate": 1.8090169943749477e-05,
-      "loss": 0.0083,
-      "num_tokens": 1880643.0,
-      "reward": 0.6052049398422241,
-      "reward_std": 0.109284907579422,
-      "rewards/cross_solution_unittest/mean": 0.6052048802375793,
-      "rewards/cross_solution_unittest/std": 0.20577581226825714,
+      "completions/max_length": 1088.0,
+      "completions/max_terminated_length": 1088.0,
+      "completions/mean_length": 493.30731201171875,
+      "completions/mean_terminated_length": 493.30731201171875,
+      "completions/min_length": 214.0,
+      "completions/min_terminated_length": 214.0,
+      "entropy": 0.0809478759765625,
+      "epoch": 0.08212147134302823,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.026688260957598686,
+      "kl": 0.00023829936981201172,
+      "learning_rate": 1.4545454545454546e-05,
+      "loss": -0.0055,
+      "num_tokens": 3802052.0,
+      "reward": 0.5806035399436951,
+      "reward_std": 0.07414089143276215,
+      "rewards/cross_solution_unittest/mean": 0.5806035399436951,
+      "rewards/cross_solution_unittest/std": 0.09448928385972977,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 9
@@ -277,24 +277,24 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -5.0,
-      "completions/max_length": 945.0,
-      "completions/max_terminated_length": 945.0,
-      "completions/mean_length": 495.9114685058594,
-      "completions/mean_terminated_length": 495.9114685058594,
-      "completions/min_length": 237.0,
-      "completions/min_terminated_length": 237.0,
-      "entropy": 0.07415771484375,
-      "epoch": 0.36446469248291574,
-      "frac_reward_zero_std": 0.0625,
-      "grad_norm": 0.032062605023384094,
-      "kl": 0.0002478361129760742,
-      "learning_rate": 1.7289686274214116e-05,
-      "loss": -0.0011,
-      "num_tokens": 2071294.0,
-      "reward": 0.5925840735435486,
-      "reward_std": 0.11311423778533936,
-      "rewards/cross_solution_unittest/mean": 0.5925840735435486,
-      "rewards/cross_solution_unittest/std": 0.19814850389957428,
+      "completions/max_length": 939.0,
+      "completions/max_terminated_length": 939.0,
+      "completions/mean_length": 515.5573120117188,
+      "completions/mean_terminated_length": 515.5573120117188,
+      "completions/min_length": 277.0,
+      "completions/min_terminated_length": 277.0,
+      "entropy": 0.0720977783203125,
+      "epoch": 0.09124607927003137,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.02697775699198246,
+      "kl": 0.00022214651107788086,
+      "learning_rate": 1.6363636363636366e-05,
+      "loss": -0.0039,
+      "num_tokens": 4250610.0,
+      "reward": 0.6249701380729675,
+      "reward_std": 0.07456190884113312,
+      "rewards/cross_solution_unittest/mean": 0.6249701380729675,
+      "rewards/cross_solution_unittest/std": 0.11200571060180664,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 10
@@ -305,25 +305,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -5.0,
-      "completions/max_length": 1178.0,
-      "completions/max_terminated_length": 1178.0,
-      "completions/mean_length": 509.1614685058594,
-      "completions/mean_terminated_length": 509.1614685058594,
-      "completions/min_length": 270.0,
-      "completions/min_terminated_length": 270.0,
-      "entropy": 0.073944091796875,
-      "epoch": 0.4009111617312073,
-      "frac_reward_zero_std": 0.0625,
-      "grad_norm": 0.030858976766467094,
-      "kl": 0.00023108720779418945,
-      "learning_rate": 1.63742398974869e-05,
-      "loss": 0.002,
-      "num_tokens": 2273865.0,
-      "reward": 0.5973134636878967,
-      "reward_std": 0.10415109246969223,
-      "rewards/cross_solution_unittest/mean": 0.5973134636878967,
-      "rewards/cross_solution_unittest/std": 0.21376842260360718,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1229.0,
+      "completions/mean_length": 561.9869995117188,
+      "completions/mean_terminated_length": 554.0966186523438,
+      "completions/min_length": 296.0,
+      "completions/min_terminated_length": 296.0,
+      "entropy": 0.0686187744140625,
+      "epoch": 0.1003706871970345,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.023875204846262932,
+      "kl": 0.00022393465042114258,
+      "learning_rate": 1.8181818181818182e-05,
+      "loss": -0.0013,
+      "num_tokens": 4686669.0,
+      "reward": 0.5655300617218018,
+      "reward_std": 0.08550869673490524,
+      "rewards/cross_solution_unittest/mean": 0.5655300617218018,
+      "rewards/cross_solution_unittest/std": 0.12360301613807678,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 11
@@ -335,24 +335,24 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -5.0,
-      "completions/max_length": 859.0,
-      "completions/max_terminated_length": 859.0,
-      "completions/mean_length": 487.49481201171875,
-      "completions/mean_terminated_length": 487.49481201171875,
-      "completions/min_length": 259.0,
-      "completions/min_terminated_length": 259.0,
-      "entropy": 0.081573486328125,
-      "epoch": 0.43735763097949887,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.035162851214408875,
-      "kl": 0.00028896331787109375,
-      "learning_rate": 1.5358267949789968e-05,
-      "loss": 0.0162,
-      "num_tokens": 2472554.0,
-      "reward": 0.5868850946426392,
-      "reward_std": 0.11655251681804657,
-      "rewards/cross_solution_unittest/mean": 0.5868851542472839,
-      "rewards/cross_solution_unittest/std": 0.2154010534286499,
+      "completions/max_length": 995.0,
+      "completions/max_terminated_length": 995.0,
+      "completions/mean_length": 500.07293701171875,
+      "completions/mean_terminated_length": 500.07293701171875,
+      "completions/min_length": 213.0,
+      "completions/min_terminated_length": 213.0,
+      "entropy": 0.0841522216796875,
+      "epoch": 0.10949529512403763,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.031900256872177124,
+      "kl": 0.0002987980842590332,
+      "learning_rate": 2e-05,
+      "loss": 0.0048,
+      "num_tokens": 5114985.0,
+      "reward": 0.628701388835907,
+      "reward_std": 0.0890752300620079,
+      "rewards/cross_solution_unittest/mean": 0.628701388835907,
+      "rewards/cross_solution_unittest/std": 0.13637541234493256,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 12
@@ -364,24 +364,24 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -5.0,
-      "completions/max_length": 1389.0,
-      "completions/max_terminated_length": 1389.0,
-      "completions/mean_length": 529.6041870117188,
-      "completions/mean_terminated_length": 529.6041870117188,
-      "completions/min_length": 217.0,
-      "completions/min_terminated_length": 217.0,
-      "entropy": 0.07177734375,
-      "epoch": 0.47380410022779046,
-      "frac_reward_zero_std": 0.1041666716337204,
-      "grad_norm": 0.029409615322947502,
-      "kl": 0.00025135278701782227,
-      "learning_rate": 1.4257792915650728e-05,
-      "loss": 0.0091,
-      "num_tokens": 2682176.0,
-      "reward": 0.5738339424133301,
-      "reward_std": 0.1161453127861023,
-      "rewards/cross_solution_unittest/mean": 0.5738338828086853,
-      "rewards/cross_solution_unittest/std": 0.22482098639011383,
+      "completions/max_length": 1188.0,
+      "completions/max_terminated_length": 1188.0,
+      "completions/mean_length": 525.2890625,
+      "completions/mean_terminated_length": 525.2890625,
+      "completions/min_length": 243.0,
+      "completions/min_terminated_length": 243.0,
+      "entropy": 0.07344818115234375,
+      "epoch": 0.11861990305104078,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0264806617051363,
+      "kl": 0.0002524852752685547,
+      "learning_rate": 1.9994965423831853e-05,
+      "loss": 0.0017,
+      "num_tokens": 5521312.0,
+      "reward": 0.5788325071334839,
+      "reward_std": 0.06292694807052612,
+      "rewards/cross_solution_unittest/mean": 0.5788324475288391,
+      "rewards/cross_solution_unittest/std": 0.10130160301923752,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 13
@@ -392,25 +392,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -5.0,
-      "completions/max_length": 959.0,
-      "completions/max_terminated_length": 959.0,
-      "completions/mean_length": 504.6614685058594,
-      "completions/mean_terminated_length": 504.6614685058594,
+      "completions/clipped_ratio": -4.859375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 2440.0,
+      "completions/mean_length": 662.984375,
+      "completions/mean_terminated_length": 592.8800048828125,
       "completions/min_length": 280.0,
       "completions/min_terminated_length": 280.0,
-      "entropy": 0.075927734375,
-      "epoch": 0.510250569476082,
+      "entropy": 0.07053375244140625,
+      "epoch": 0.1277445109780439,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.03111596405506134,
-      "kl": 0.00023740530014038086,
-      "learning_rate": 1.3090169943749475e-05,
-      "loss": -0.0007,
-      "num_tokens": 2885113.0,
-      "reward": 0.5357317328453064,
-      "reward_std": 0.1203121468424797,
-      "rewards/cross_solution_unittest/mean": 0.5357317328453064,
-      "rewards/cross_solution_unittest/std": 0.2490031123161316,
+      "grad_norm": 0.025386298075318336,
+      "kl": 0.00023430585861206055,
+      "learning_rate": 1.9979866764718846e-05,
+      "loss": -0.0055,
+      "num_tokens": 6009574.0,
+      "reward": 0.5900708436965942,
+      "reward_std": 0.0674651488661766,
+      "rewards/cross_solution_unittest/mean": 0.5900708436965942,
+      "rewards/cross_solution_unittest/std": 0.10158919543027878,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 14
@@ -421,25 +421,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -5.0,
-      "completions/max_length": 926.0,
-      "completions/max_terminated_length": 926.0,
-      "completions/mean_length": 502.9739685058594,
-      "completions/mean_terminated_length": 502.9739685058594,
-      "completions/min_length": 243.0,
-      "completions/min_terminated_length": 243.0,
-      "entropy": 0.08111572265625,
-      "epoch": 0.5466970387243736,
-      "frac_reward_zero_std": 0.02083333395421505,
-      "grad_norm": 0.031033573672175407,
-      "kl": 0.00027930736541748047,
-      "learning_rate": 1.187381314585725e-05,
-      "loss": -0.0008,
-      "num_tokens": 3098294.0,
-      "reward": 0.5646395683288574,
-      "reward_std": 0.10859303176403046,
-      "rewards/cross_solution_unittest/mean": 0.5646395087242126,
-      "rewards/cross_solution_unittest/std": 0.19871041178703308,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 830.0,
+      "completions/mean_length": 475.04168701171875,
+      "completions/mean_terminated_length": 458.764404296875,
+      "completions/min_length": 254.0,
+      "completions/min_terminated_length": 254.0,
+      "entropy": 0.0851287841796875,
+      "epoch": 0.13686911890504705,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.02971607819199562,
+      "kl": 0.00028899312019348145,
+      "learning_rate": 1.9954719225730847e-05,
+      "loss": 0.0129,
+      "num_tokens": 6357330.0,
+      "reward": 0.5344895124435425,
+      "reward_std": 0.08001528680324554,
+      "rewards/cross_solution_unittest/mean": 0.5344895720481873,
+      "rewards/cross_solution_unittest/std": 0.13401487469673157,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 15
@@ -450,25 +450,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -5.0,
-      "completions/max_length": 1039.0,
-      "completions/max_terminated_length": 1039.0,
-      "completions/mean_length": 512.5729370117188,
-      "completions/mean_terminated_length": 512.5729370117188,
-      "completions/min_length": 223.0,
-      "completions/min_terminated_length": 223.0,
-      "entropy": 0.075897216796875,
-      "epoch": 0.5831435079726651,
-      "frac_reward_zero_std": 0.0833333358168602,
-      "grad_norm": 0.02894579991698265,
-      "kl": 0.0002409219741821289,
-      "learning_rate": 1.0627905195293135e-05,
-      "loss": 0.0065,
-      "num_tokens": 3304324.0,
-      "reward": 0.5695656538009644,
-      "reward_std": 0.10569658130407333,
-      "rewards/cross_solution_unittest/mean": 0.5695656538009644,
-      "rewards/cross_solution_unittest/std": 0.20740273594856262,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 3316.0,
+      "completions/mean_length": 538.6823120117188,
+      "completions/mean_terminated_length": 530.7310791015625,
+      "completions/min_length": 224.0,
+      "completions/min_terminated_length": 224.0,
+      "entropy": 0.0736846923828125,
+      "epoch": 0.1459937268320502,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.027988266199827194,
+      "kl": 0.00025466084480285645,
+      "learning_rate": 1.9919548128307954e-05,
+      "loss": -0.008,
+      "num_tokens": 6763168.0,
+      "reward": 0.5849003195762634,
+      "reward_std": 0.04933098703622818,
+      "rewards/cross_solution_unittest/mean": 0.5849003195762634,
+      "rewards/cross_solution_unittest/std": 0.1075245663523674,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 16
@@ -480,24 +480,24 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -5.0,
-      "completions/max_length": 898.0,
-      "completions/max_terminated_length": 898.0,
-      "completions/mean_length": 506.38543701171875,
-      "completions/mean_terminated_length": 506.38543701171875,
-      "completions/min_length": 249.0,
-      "completions/min_terminated_length": 249.0,
-      "entropy": 0.0773773193359375,
-      "epoch": 0.6195899772209568,
-      "frac_reward_zero_std": 0.02083333395421505,
-      "grad_norm": 0.03125438094139099,
-      "kl": 0.00026804208755493164,
-      "learning_rate": 9.372094804706867e-06,
-      "loss": 0.0019,
-      "num_tokens": 3520562.0,
-      "reward": 0.609571099281311,
-      "reward_std": 0.09919850528240204,
-      "rewards/cross_solution_unittest/mean": 0.6095710396766663,
-      "rewards/cross_solution_unittest/std": 0.20345403254032135,
+      "completions/max_length": 2246.0,
+      "completions/max_terminated_length": 2246.0,
+      "completions/mean_length": 522.3177490234375,
+      "completions/mean_terminated_length": 522.3177490234375,
+      "completions/min_length": 296.0,
+      "completions/min_terminated_length": 296.0,
+      "entropy": 0.078094482421875,
+      "epoch": 0.15511833475905332,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.025351958349347115,
+      "kl": 0.00026172399520874023,
+      "learning_rate": 1.9874388886763944e-05,
+      "loss": -0.0062,
+      "num_tokens": 7176030.0,
+      "reward": 0.5198549032211304,
+      "reward_std": 0.07214348018169403,
+      "rewards/cross_solution_unittest/mean": 0.5198548436164856,
+      "rewards/cross_solution_unittest/std": 0.14567282795906067,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 17
@@ -508,25 +508,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -4.96875,
-      "completions/max_length": 3584.0,
-      "completions/max_terminated_length": 1040.0,
-      "completions/mean_length": 550.796875,
-      "completions/mean_terminated_length": 534.916259765625,
-      "completions/min_length": 270.0,
-      "completions/min_terminated_length": 270.0,
-      "entropy": 0.072509765625,
-      "epoch": 0.6560364464692483,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.03002159111201763,
-      "kl": 0.00025850534439086914,
-      "learning_rate": 8.126186854142752e-06,
-      "loss": -0.0039,
-      "num_tokens": 3739721.0,
-      "reward": 0.6008386015892029,
-      "reward_std": 0.11992423236370087,
-      "rewards/cross_solution_unittest/mean": 0.6008385419845581,
-      "rewards/cross_solution_unittest/std": 0.2292270064353943,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1374.0,
+      "completions/max_terminated_length": 1374.0,
+      "completions/mean_length": 536.2578125,
+      "completions/mean_terminated_length": 536.2578125,
+      "completions/min_length": 233.0,
+      "completions/min_terminated_length": 233.0,
+      "entropy": 0.07436370849609375,
+      "epoch": 0.16424294268605646,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.027073806151747704,
+      "kl": 0.0002745389938354492,
+      "learning_rate": 1.9819286972627066e-05,
+      "loss": 0.0116,
+      "num_tokens": 7612777.0,
+      "reward": 0.6148377656936646,
+      "reward_std": 0.05920207127928734,
+      "rewards/cross_solution_unittest/mean": 0.6148377060890198,
+      "rewards/cross_solution_unittest/std": 0.10591181367635727,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 18
@@ -537,25 +537,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -5.0,
-      "completions/max_length": 1008.0,
-      "completions/max_terminated_length": 1008.0,
-      "completions/mean_length": 518.234375,
-      "completions/mean_terminated_length": 518.234375,
-      "completions/min_length": 197.0,
-      "completions/min_terminated_length": 197.0,
-      "entropy": 0.0797119140625,
-      "epoch": 0.6924829157175398,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.03135575354099274,
-      "kl": 0.00027000904083251953,
-      "learning_rate": 6.909830056250527e-06,
-      "loss": -0.01,
-      "num_tokens": 3935738.0,
-      "reward": 0.5821563005447388,
-      "reward_std": 0.11166616529226303,
-      "rewards/cross_solution_unittest/mean": 0.582156240940094,
-      "rewards/cross_solution_unittest/std": 0.20721960067749023,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1357.0,
+      "completions/mean_length": 507.0546875,
+      "completions/mean_terminated_length": 499.0209045410156,
+      "completions/min_length": 234.0,
+      "completions/min_terminated_length": 234.0,
+      "entropy": 0.0801239013671875,
+      "epoch": 0.17336755061305958,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.027948658913373947,
+      "kl": 0.00029531121253967285,
+      "learning_rate": 1.9754297868854075e-05,
+      "loss": -0.0075,
+      "num_tokens": 8027914.0,
+      "reward": 0.5844996571540833,
+      "reward_std": 0.07097827643156052,
+      "rewards/cross_solution_unittest/mean": 0.5844996571540833,
+      "rewards/cross_solution_unittest/std": 0.09445778280496597,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 19
@@ -566,25 +566,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -4.9375,
+      "completions/clipped_ratio": -4.984375,
       "completions/max_length": 3584.0,
-      "completions/max_terminated_length": 1872.0,
-      "completions/mean_length": 546.28125,
-      "completions/mean_terminated_length": 514.3052978515625,
-      "completions/min_length": 272.0,
-      "completions/min_terminated_length": 272.0,
-      "entropy": 0.07513427734375,
-      "epoch": 0.7289293849658315,
-      "frac_reward_zero_std": 0.02083333395421505,
-      "grad_norm": 0.03126757591962814,
-      "kl": 0.00025522708892822266,
-      "learning_rate": 5.742207084349274e-06,
-      "loss": -0.0025,
-      "num_tokens": 4145042.0,
-      "reward": 0.5788866281509399,
-      "reward_std": 0.12265484035015106,
-      "rewards/cross_solution_unittest/mean": 0.5788866281509399,
-      "rewards/cross_solution_unittest/std": 0.2369561791419983,
+      "completions/max_terminated_length": 1284.0,
+      "completions/mean_length": 542.0234375,
+      "completions/mean_terminated_length": 534.0809326171875,
+      "completions/min_length": 298.0,
+      "completions/min_terminated_length": 298.0,
+      "entropy": 0.07269287109375,
+      "epoch": 0.18249215854006273,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.028256384655833244,
+      "kl": 0.0003152787685394287,
+      "learning_rate": 1.9679487013963566e-05,
+      "loss": 0.0065,
+      "num_tokens": 8484811.0,
+      "reward": 0.6114683151245117,
+      "reward_std": 0.06493355333805084,
+      "rewards/cross_solution_unittest/mean": 0.6114682555198669,
+      "rewards/cross_solution_unittest/std": 0.09383873641490936,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 20
@@ -596,24 +596,24 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -5.0,
-      "completions/max_length": 1437.0,
-      "completions/max_terminated_length": 1437.0,
-      "completions/mean_length": 495.43231201171875,
-      "completions/mean_terminated_length": 495.43231201171875,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
-      "entropy": 0.07891845703125,
-      "epoch": 0.765375854214123,
-      "frac_reward_zero_std": 0.02083333395421505,
-      "grad_norm": 0.032083988189697266,
-      "kl": 0.00027364492416381836,
-      "learning_rate": 4.641732050210032e-06,
-      "loss": 0.0086,
-      "num_tokens": 4336333.0,
-      "reward": 0.59117591381073,
-      "reward_std": 0.12750200927257538,
-      "rewards/cross_solution_unittest/mean": 0.5911758542060852,
-      "rewards/cross_solution_unittest/std": 0.22702106833457947,
+      "completions/max_length": 989.0,
+      "completions/max_terminated_length": 989.0,
+      "completions/mean_length": 483.76043701171875,
+      "completions/mean_terminated_length": 483.76043701171875,
+      "completions/min_length": 197.0,
+      "completions/min_terminated_length": 197.0,
+      "entropy": 0.0761260986328125,
+      "epoch": 0.19161676646706588,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.027799971401691437,
+      "kl": 0.00030609965324401855,
+      "learning_rate": 1.9594929736144978e-05,
+      "loss": 0.0025,
+      "num_tokens": 8869807.0,
+      "reward": 0.6294624209403992,
+      "reward_std": 0.06047224625945091,
+      "rewards/cross_solution_unittest/mean": 0.6294623613357544,
+      "rewards/cross_solution_unittest/std": 0.10432589054107666,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 21
@@ -624,25 +624,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -5.0,
-      "completions/max_length": 1204.0,
-      "completions/max_terminated_length": 1204.0,
-      "completions/mean_length": 500.3177185058594,
-      "completions/mean_terminated_length": 500.3177185058594,
-      "completions/min_length": 234.0,
-      "completions/min_terminated_length": 234.0,
-      "entropy": 0.0772857666015625,
-      "epoch": 0.8018223234624146,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.029832610860466957,
-      "kl": 0.0002658963203430176,
-      "learning_rate": 3.625760102513103e-06,
-      "loss": -0.0039,
-      "num_tokens": 4544172.0,
-      "reward": 0.5655040740966797,
-      "reward_std": 0.10794761031866074,
-      "rewards/cross_solution_unittest/mean": 0.5655040740966797,
-      "rewards/cross_solution_unittest/std": 0.22446972131729126,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 951.0,
+      "completions/mean_length": 518.3724365234375,
+      "completions/mean_terminated_length": 510.3681640625,
+      "completions/min_length": 224.0,
+      "completions/min_terminated_length": 224.0,
+      "entropy": 0.07337188720703125,
+      "epoch": 0.200741374394069,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.029701806604862213,
+      "kl": 0.00030997395515441895,
+      "learning_rate": 1.9500711177409456e-05,
+      "loss": -0.0053,
+      "num_tokens": 9251294.0,
+      "reward": 0.581719160079956,
+      "reward_std": 0.07320186495780945,
+      "rewards/cross_solution_unittest/mean": 0.581719160079956,
+      "rewards/cross_solution_unittest/std": 0.16921019554138184,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 22
@@ -653,25 +653,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -5.0,
-      "completions/max_length": 864.0,
-      "completions/max_terminated_length": 864.0,
-      "completions/mean_length": 509.30731201171875,
-      "completions/mean_terminated_length": 509.30731201171875,
-      "completions/min_length": 243.0,
-      "completions/min_terminated_length": 243.0,
-      "entropy": 0.0737457275390625,
-      "epoch": 0.8382687927107062,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1624.0,
+      "completions/mean_length": 529.96875,
+      "completions/mean_terminated_length": 521.9948120117188,
+      "completions/min_length": 272.0,
+      "completions/min_terminated_length": 272.0,
+      "entropy": 0.0753173828125,
+      "epoch": 0.20986598232107215,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.030700158327817917,
-      "kl": 0.0002843141555786133,
-      "learning_rate": 2.7103137257858867e-06,
-      "loss": 0.0074,
-      "num_tokens": 4752069.0,
-      "reward": 0.5879608392715454,
-      "reward_std": 0.11106596887111664,
-      "rewards/cross_solution_unittest/mean": 0.5879607796669006,
-      "rewards/cross_solution_unittest/std": 0.2123921662569046,
+      "grad_norm": 0.027412766590714455,
+      "kl": 0.0003858804702758789,
+      "learning_rate": 1.9396926207859085e-05,
+      "loss": -0.0033,
+      "num_tokens": 9705346.0,
+      "reward": 0.5391039848327637,
+      "reward_std": 0.06848187744617462,
+      "rewards/cross_solution_unittest/mean": 0.5391039848327637,
+      "rewards/cross_solution_unittest/std": 0.1495925337076187,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 23
@@ -682,25 +682,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -4.9375,
-      "completions/max_length": 3584.0,
-      "completions/max_terminated_length": 908.0,
-      "completions/mean_length": 567.7552490234375,
-      "completions/mean_terminated_length": 536.0053100585938,
-      "completions/min_length": 284.0,
-      "completions/min_terminated_length": 284.0,
-      "entropy": 0.069793701171875,
-      "epoch": 0.8747152619589977,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.028657682240009308,
-      "kl": 0.00025784969329833984,
-      "learning_rate": 1.9098300562505266e-06,
-      "loss": -0.0067,
-      "num_tokens": 4971092.0,
-      "reward": 0.5960028171539307,
-      "reward_std": 0.10218709707260132,
-      "rewards/cross_solution_unittest/mean": 0.5960028171539307,
-      "rewards/cross_solution_unittest/std": 0.19247692823410034,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 895.0,
+      "completions/max_terminated_length": 895.0,
+      "completions/mean_length": 508.359375,
+      "completions/mean_terminated_length": 508.359375,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
+      "entropy": 0.07342529296875,
+      "epoch": 0.21899059024807527,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.025877749547362328,
+      "kl": 0.00034868717193603516,
+      "learning_rate": 1.9283679330160726e-05,
+      "loss": 0.0015,
+      "num_tokens": 10126856.0,
+      "reward": 0.601729691028595,
+      "reward_std": 0.08979599177837372,
+      "rewards/cross_solution_unittest/mean": 0.6017296314239502,
+      "rewards/cross_solution_unittest/std": 0.13213877379894257,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 24
@@ -712,24 +712,24 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -5.0,
-      "completions/max_length": 950.0,
-      "completions/max_terminated_length": 950.0,
-      "completions/mean_length": 494.65625,
-      "completions/mean_terminated_length": 494.65625,
-      "completions/min_length": 279.0,
-      "completions/min_terminated_length": 279.0,
-      "entropy": 0.078704833984375,
-      "epoch": 0.9111617312072893,
-      "frac_reward_zero_std": 0.0416666679084301,
-      "grad_norm": 0.03116031549870968,
-      "kl": 0.00027370452880859375,
-      "learning_rate": 1.2369331995613664e-06,
-      "loss": 0.0021,
-      "num_tokens": 5166194.0,
-      "reward": 0.554139256477356,
-      "reward_std": 0.0889216810464859,
-      "rewards/cross_solution_unittest/mean": 0.554139256477356,
-      "rewards/cross_solution_unittest/std": 0.1982363760471344,
+      "completions/max_length": 1476.0,
+      "completions/max_terminated_length": 1476.0,
+      "completions/mean_length": 500.75262451171875,
+      "completions/mean_terminated_length": 500.75262451171875,
+      "completions/min_length": 259.0,
+      "completions/min_terminated_length": 259.0,
+      "entropy": 0.0807342529296875,
+      "epoch": 0.22811519817507842,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.030456064268946648,
+      "kl": 0.00040835142135620117,
+      "learning_rate": 1.9161084574320696e-05,
+      "loss": 0.0033,
+      "num_tokens": 10534033.0,
+      "reward": 0.6099452972412109,
+      "reward_std": 0.05933503434062004,
+      "rewards/cross_solution_unittest/mean": 0.6099452376365662,
+      "rewards/cross_solution_unittest/std": 0.08372820913791656,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 25
@@ -740,25 +740,25 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -5.0,
-      "completions/max_length": 767.0,
-      "completions/max_terminated_length": 767.0,
-      "completions/mean_length": 497.953125,
-      "completions/mean_terminated_length": 497.953125,
-      "completions/min_length": 262.0,
-      "completions/min_terminated_length": 262.0,
-      "entropy": 0.073150634765625,
-      "epoch": 0.9476082004555809,
-      "frac_reward_zero_std": 0.02083333395421505,
-      "grad_norm": 0.029424885287880898,
-      "kl": 0.00023937225341796875,
-      "learning_rate": 7.022351411174866e-07,
-      "loss": -0.0041,
-      "num_tokens": 5368559.0,
-      "reward": 0.5750547647476196,
-      "reward_std": 0.11895763874053955,
-      "rewards/cross_solution_unittest/mean": 0.5750547647476196,
-      "rewards/cross_solution_unittest/std": 0.18430344760417938,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1958.0,
+      "completions/mean_length": 536.2213745117188,
+      "completions/mean_terminated_length": 520.264404296875,
+      "completions/min_length": 253.0,
+      "completions/min_terminated_length": 253.0,
+      "entropy": 0.077789306640625,
+      "epoch": 0.23723980610208156,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.027375157922506332,
+      "kl": 0.0004361867904663086,
+      "learning_rate": 1.9029265382866216e-05,
+      "loss": -0.0071,
+      "num_tokens": 10909678.0,
+      "reward": 0.5840487480163574,
+      "reward_std": 0.07150473445653915,
+      "rewards/cross_solution_unittest/mean": 0.5840487480163574,
+      "rewards/cross_solution_unittest/std": 0.12009274214506149,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 26
@@ -770,50 +770,2428 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -5.0,
-      "completions/max_length": 986.0,
-      "completions/max_terminated_length": 986.0,
-      "completions/mean_length": 518.8073120117188,
-      "completions/mean_terminated_length": 518.8073120117188,
-      "completions/min_length": 270.0,
-      "completions/min_terminated_length": 270.0,
-      "entropy": 0.073272705078125,
-      "epoch": 0.9840546697038725,
+      "completions/max_length": 838.0,
+      "completions/max_terminated_length": 838.0,
+      "completions/mean_length": 471.58856201171875,
+      "completions/mean_terminated_length": 471.58856201171875,
+      "completions/min_length": 230.0,
+      "completions/min_terminated_length": 230.0,
+      "entropy": 0.0792694091796875,
+      "epoch": 0.24636441402908468,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 0.030289024114608765,
-      "kl": 0.00026422739028930664,
-      "learning_rate": 3.1416838871368925e-07,
-      "loss": 0.0135,
-      "num_tokens": 5576354.0,
-      "reward": 0.5601072311401367,
-      "reward_std": 0.10010793805122375,
-      "rewards/cross_solution_unittest/mean": 0.5601072311401367,
-      "rewards/cross_solution_unittest/std": 0.18709710240364075,
+      "grad_norm": 0.027460293844342232,
+      "kl": 0.000461578369140625,
+      "learning_rate": 1.8888354486549238e-05,
+      "loss": -0.0034,
+      "num_tokens": 11287180.0,
+      "reward": 0.5752553939819336,
+      "reward_std": 0.06591899693012238,
+      "rewards/cross_solution_unittest/mean": 0.5752554535865784,
+      "rewards/cross_solution_unittest/std": 0.12003585696220398,
       "rewards/format_reward/mean": 0.0,
       "rewards/format_reward/std": 0.0,
       "step": 27
     },
     {
-      "epoch": 0.9840546697038725,
-      "step": 27,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1051.0,
+      "completions/max_terminated_length": 1051.0,
+      "completions/mean_length": 511.953125,
+      "completions/mean_terminated_length": 511.953125,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
+      "entropy": 0.075927734375,
+      "epoch": 0.2554890219560878,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.029021723195910454,
+      "kl": 0.0005039572715759277,
+      "learning_rate": 1.873849377069785e-05,
+      "loss": -0.0042,
+      "num_tokens": 11722918.0,
+      "reward": 0.5585534572601318,
+      "reward_std": 0.08278067409992218,
+      "rewards/cross_solution_unittest/mean": 0.5585535168647766,
+      "rewards/cross_solution_unittest/std": 0.15765663981437683,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 28
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1088.0,
+      "completions/mean_length": 498.6302185058594,
+      "completions/mean_terminated_length": 490.5744323730469,
+      "completions/min_length": 220.0,
+      "completions/min_terminated_length": 220.0,
+      "entropy": 0.08233642578125,
+      "epoch": 0.26461362988309095,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.029287124052643776,
+      "kl": 0.0005979537963867188,
+      "learning_rate": 1.8579834132349773e-05,
+      "loss": 0.016,
+      "num_tokens": 12122528.0,
+      "reward": 0.5899044275283813,
+      "reward_std": 0.060223691165447235,
+      "rewards/cross_solution_unittest/mean": 0.5899044871330261,
+      "rewards/cross_solution_unittest/std": 0.09450040012598038,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 29
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.953125,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 2053.0,
+      "completions/mean_length": 553.2057495117188,
+      "completions/mean_terminated_length": 529.3411865234375,
+      "completions/min_length": 261.0,
+      "completions/min_terminated_length": 261.0,
+      "entropy": 0.075469970703125,
+      "epoch": 0.2737382378100941,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.028479870408773422,
+      "kl": 0.0005797147750854492,
+      "learning_rate": 1.8412535328311813e-05,
+      "loss": 0.0051,
+      "num_tokens": 12555555.0,
+      "reward": 0.5730323791503906,
+      "reward_std": 0.11489957571029663,
+      "rewards/cross_solution_unittest/mean": 0.5730323195457458,
+      "rewards/cross_solution_unittest/std": 0.18645499646663666,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 30
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 972.0,
+      "completions/max_terminated_length": 972.0,
+      "completions/mean_length": 473.4453125,
+      "completions/mean_terminated_length": 473.4453125,
+      "completions/min_length": 217.0,
+      "completions/min_terminated_length": 217.0,
+      "entropy": 0.084716796875,
+      "epoch": 0.28286284573709725,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.02932862937450409,
+      "kl": 0.0007239580154418945,
+      "learning_rate": 1.8236765814298328e-05,
+      "loss": 0.0069,
+      "num_tokens": 12922190.0,
+      "reward": 0.6376906633377075,
+      "reward_std": 0.07534094899892807,
+      "rewards/cross_solution_unittest/mean": 0.6376906037330627,
+      "rewards/cross_solution_unittest/std": 0.11291607469320297,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 31
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1047.0,
+      "completions/max_terminated_length": 1047.0,
+      "completions/mean_length": 485.9609375,
+      "completions/mean_terminated_length": 485.9609375,
+      "completions/min_length": 226.0,
+      "completions/min_terminated_length": 226.0,
+      "entropy": 0.0768280029296875,
+      "epoch": 0.2919874536641004,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0313863605260849,
+      "kl": 0.000737309455871582,
+      "learning_rate": 1.8052702575310588e-05,
+      "loss": -0.0043,
+      "num_tokens": 13378943.0,
+      "reward": 0.6212249994277954,
+      "reward_std": 0.08826343715190887,
+      "rewards/cross_solution_unittest/mean": 0.6212249398231506,
+      "rewards/cross_solution_unittest/std": 0.1459883153438568,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 32
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1008.0,
+      "completions/mean_length": 500.5911560058594,
+      "completions/mean_terminated_length": 492.54046630859375,
+      "completions/min_length": 225.0,
+      "completions/min_terminated_length": 225.0,
+      "entropy": 0.0904998779296875,
+      "epoch": 0.3011120615911035,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.028566641733050346,
+      "kl": 0.0008232593536376953,
+      "learning_rate": 1.7860530947427878e-05,
+      "loss": -0.0054,
+      "num_tokens": 13765758.0,
+      "reward": 0.6300634741783142,
+      "reward_std": 0.07626457512378693,
+      "rewards/cross_solution_unittest/mean": 0.6300634741783142,
+      "rewards/cross_solution_unittest/std": 0.11189531534910202,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 33
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 931.0,
+      "completions/max_terminated_length": 931.0,
+      "completions/mean_length": 505.2109375,
+      "completions/mean_terminated_length": 505.2109375,
+      "completions/min_length": 254.0,
+      "completions/min_terminated_length": 254.0,
+      "entropy": 0.0770263671875,
+      "epoch": 0.31023666951810663,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.02723107673227787,
+      "kl": 0.0009553432464599609,
+      "learning_rate": 1.766044443118978e-05,
+      "loss": 0.0036,
+      "num_tokens": 14200039.0,
+      "reward": 0.6032774448394775,
+      "reward_std": 0.05867696925997734,
+      "rewards/cross_solution_unittest/mean": 0.6032774448394775,
+      "rewards/cross_solution_unittest/std": 0.0986337959766388,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 34
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 2461.0,
+      "completions/mean_length": 516.0833740234375,
+      "completions/mean_terminated_length": 508.0731201171875,
+      "completions/min_length": 231.0,
+      "completions/min_terminated_length": 231.0,
+      "entropy": 0.07989501953125,
+      "epoch": 0.3193612774451098,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03081453964114189,
+      "kl": 0.0009756088256835938,
+      "learning_rate": 1.745264449675755e-05,
+      "loss": -0.0036,
+      "num_tokens": 14588483.0,
+      "reward": 0.5146794319152832,
+      "reward_std": 0.07540898025035858,
+      "rewards/cross_solution_unittest/mean": 0.514679491519928,
+      "rewards/cross_solution_unittest/std": 0.1279023289680481,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 35
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 947.0,
+      "completions/max_terminated_length": 947.0,
+      "completions/mean_length": 523.4557495117188,
+      "completions/mean_terminated_length": 523.4557495117188,
+      "completions/min_length": 319.0,
+      "completions/min_terminated_length": 319.0,
+      "entropy": 0.079742431640625,
+      "epoch": 0.32848588537211293,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0302255991846323,
+      "kl": 0.0010718107223510742,
+      "learning_rate": 1.72373403810507e-05,
+      "loss": 0.0038,
+      "num_tokens": 15012890.0,
+      "reward": 0.5417085886001587,
+      "reward_std": 0.07886814326047897,
+      "rewards/cross_solution_unittest/mean": 0.5417085289955139,
+      "rewards/cross_solution_unittest/std": 0.14164547622203827,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 36
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1876.0,
+      "completions/max_terminated_length": 1876.0,
+      "completions/mean_length": 514.140625,
+      "completions/mean_terminated_length": 514.140625,
+      "completions/min_length": 227.0,
+      "completions/min_terminated_length": 227.0,
+      "entropy": 0.0728759765625,
+      "epoch": 0.3376104932991161,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.030379414558410645,
+      "kl": 0.0013577938079833984,
+      "learning_rate": 1.7014748877063212e-05,
+      "loss": -0.0051,
+      "num_tokens": 15419892.0,
+      "reward": 0.6429738402366638,
+      "reward_std": 0.07094022631645203,
+      "rewards/cross_solution_unittest/mean": 0.6429738402366638,
+      "rewards/cross_solution_unittest/std": 0.1320594996213913,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 37
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 887.0,
+      "completions/max_terminated_length": 887.0,
+      "completions/mean_length": 477.8046875,
+      "completions/mean_terminated_length": 477.8046875,
+      "completions/min_length": 231.0,
+      "completions/min_terminated_length": 231.0,
+      "entropy": 0.079803466796875,
+      "epoch": 0.34673510122611917,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03055444173514843,
+      "kl": 0.0011442899703979492,
+      "learning_rate": 1.6785094115571323e-05,
+      "loss": -0.0004,
+      "num_tokens": 15801089.0,
+      "reward": 0.5747675895690918,
+      "reward_std": 0.06717974692583084,
+      "rewards/cross_solution_unittest/mean": 0.5747675895690918,
+      "rewards/cross_solution_unittest/std": 0.08518695831298828,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 38
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1270.0,
+      "completions/mean_length": 476.1770935058594,
+      "completions/mean_terminated_length": 468.06268310546875,
+      "completions/min_length": 131.0,
+      "completions/min_terminated_length": 131.0,
+      "entropy": 0.087554931640625,
+      "epoch": 0.3558597091531223,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04223545268177986,
+      "kl": 0.0014077425003051758,
+      "learning_rate": 1.6548607339452853e-05,
+      "loss": 0.0023,
+      "num_tokens": 16114473.0,
+      "reward": 0.5905721187591553,
+      "reward_std": 0.09474077075719833,
+      "rewards/cross_solution_unittest/mean": 0.5905721187591553,
+      "rewards/cross_solution_unittest/std": 0.13974814116954803,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 39
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 987.0,
+      "completions/mean_length": 491.4895935058594,
+      "completions/mean_terminated_length": 475.2984313964844,
+      "completions/min_length": 232.0,
+      "completions/min_terminated_length": 232.0,
+      "entropy": 0.07975006103515625,
+      "epoch": 0.36498431708012546,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03678172081708908,
+      "kl": 0.0016982555389404297,
+      "learning_rate": 1.6305526670845225e-05,
+      "loss": -0.0074,
+      "num_tokens": 16524457.0,
+      "reward": 0.5999099612236023,
+      "reward_std": 0.09236396849155426,
+      "rewards/cross_solution_unittest/mean": 0.5999099612236023,
+      "rewards/cross_solution_unittest/std": 0.1273658722639084,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 40
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 924.0,
+      "completions/mean_length": 501.82293701171875,
+      "completions/mean_terminated_length": 493.77545166015625,
+      "completions/min_length": 234.0,
+      "completions/min_terminated_length": 234.0,
+      "entropy": 0.08660888671875,
+      "epoch": 0.3741089250071286,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03592362627387047,
+      "kl": 0.0014951229095458984,
+      "learning_rate": 1.6056096871376667e-05,
+      "loss": -0.0111,
+      "num_tokens": 16940425.0,
+      "reward": 0.6201981902122498,
+      "reward_std": 0.07472098618745804,
+      "rewards/cross_solution_unittest/mean": 0.620198130607605,
+      "rewards/cross_solution_unittest/std": 0.12231894582509995,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 41
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1250.0,
+      "completions/max_terminated_length": 1250.0,
+      "completions/mean_length": 526.3515625,
+      "completions/mean_terminated_length": 526.3515625,
+      "completions/min_length": 262.0,
+      "completions/min_terminated_length": 262.0,
+      "entropy": 0.069305419921875,
+      "epoch": 0.38323353293413176,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.032890286296606064,
+      "kl": 0.0016560554504394531,
+      "learning_rate": 1.5800569095711983e-05,
+      "loss": -0.0149,
+      "num_tokens": 17392560.0,
+      "reward": 0.5883609652519226,
+      "reward_std": 0.09757032245397568,
+      "rewards/cross_solution_unittest/mean": 0.5883609056472778,
+      "rewards/cross_solution_unittest/std": 0.138299360871315,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 42
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 883.0,
+      "completions/mean_length": 492.0703125,
+      "completions/mean_terminated_length": 483.9974060058594,
+      "completions/min_length": 253.0,
+      "completions/min_terminated_length": 253.0,
+      "entropy": 0.0814666748046875,
+      "epoch": 0.39235814086113485,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.030933000147342682,
+      "kl": 0.0019317865371704102,
+      "learning_rate": 1.5539200638661106e-05,
+      "loss": 0.0112,
+      "num_tokens": 17737743.0,
+      "reward": 0.6191085577011108,
+      "reward_std": 0.07476722449064255,
+      "rewards/cross_solution_unittest/mean": 0.6191084980964661,
+      "rewards/cross_solution_unittest/std": 0.11829123646020889,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 43
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1413.0,
+      "completions/mean_length": 531.0286865234375,
+      "completions/mean_terminated_length": 523.0574340820312,
+      "completions/min_length": 247.0,
+      "completions/min_terminated_length": 247.0,
+      "entropy": 0.07323455810546875,
+      "epoch": 0.401482748788138,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.02944750152528286,
+      "kl": 0.0017609596252441406,
+      "learning_rate": 1.5272254676105026e-05,
+      "loss": -0.0072,
+      "num_tokens": 18154030.0,
+      "reward": 0.5842992067337036,
+      "reward_std": 0.07545933127403259,
+      "rewards/cross_solution_unittest/mean": 0.5842992067337036,
+      "rewards/cross_solution_unittest/std": 0.13810177147388458,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 44
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 938.0,
+      "completions/max_terminated_length": 938.0,
+      "completions/mean_length": 468.75,
+      "completions/mean_terminated_length": 468.75,
+      "completions/min_length": 224.0,
+      "completions/min_terminated_length": 224.0,
+      "entropy": 0.0874786376953125,
+      "epoch": 0.41060735671514115,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03713710606098175,
+      "kl": 0.002506732940673828,
+      "learning_rate": 1.5000000000000002e-05,
+      "loss": 0.0025,
+      "num_tokens": 18535066.0,
+      "reward": 0.6071228981018066,
+      "reward_std": 0.08849634975194931,
+      "rewards/cross_solution_unittest/mean": 0.6071229577064514,
+      "rewards/cross_solution_unittest/std": 0.1443415731191635,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 45
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 885.0,
+      "completions/max_terminated_length": 885.0,
+      "completions/mean_length": 507.75262451171875,
+      "completions/mean_terminated_length": 507.75262451171875,
+      "completions/min_length": 277.0,
+      "completions/min_terminated_length": 277.0,
+      "entropy": 0.07917022705078125,
+      "epoch": 0.4197319646421443,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.030308177694678307,
+      "kl": 0.0022301673889160156,
+      "learning_rate": 1.472271074772683e-05,
+      "loss": -0.0041,
+      "num_tokens": 18951727.0,
+      "reward": 0.6283596158027649,
+      "reward_std": 0.05247451364994049,
+      "rewards/cross_solution_unittest/mean": 0.6283596158027649,
+      "rewards/cross_solution_unittest/std": 0.07481828331947327,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 46
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1472.0,
+      "completions/max_terminated_length": 1472.0,
+      "completions/mean_length": 498.56512451171875,
+      "completions/mean_terminated_length": 498.56512451171875,
+      "completions/min_length": 239.0,
+      "completions/min_terminated_length": 239.0,
+      "entropy": 0.0898895263671875,
+      "epoch": 0.42885657256914744,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03285127133131027,
+      "kl": 0.0027070045471191406,
+      "learning_rate": 1.4440666126057743e-05,
+      "loss": -0.0049,
+      "num_tokens": 19368960.0,
+      "reward": 0.5909382104873657,
+      "reward_std": 0.07557514309883118,
+      "rewards/cross_solution_unittest/mean": 0.590938150882721,
+      "rewards/cross_solution_unittest/std": 0.14489254355430603,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 47
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 840.0,
+      "completions/max_terminated_length": 840.0,
+      "completions/mean_length": 501.1015625,
+      "completions/mean_terminated_length": 501.1015625,
+      "completions/min_length": 243.0,
+      "completions/min_terminated_length": 243.0,
+      "entropy": 0.0812225341796875,
+      "epoch": 0.43798118049615054,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03334600478410721,
+      "kl": 0.003113985061645508,
+      "learning_rate": 1.4154150130018867e-05,
+      "loss": 0.0134,
+      "num_tokens": 19762247.0,
+      "reward": 0.6029683351516724,
+      "reward_std": 0.07560944557189941,
+      "rewards/cross_solution_unittest/mean": 0.6029682755470276,
+      "rewards/cross_solution_unittest/std": 0.112706758081913,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 48
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.953125,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1296.0,
+      "completions/mean_length": 534.4896240234375,
+      "completions/mean_terminated_length": 510.4776916503906,
+      "completions/min_length": 276.0,
+      "completions/min_terminated_length": 276.0,
+      "entropy": 0.0760498046875,
+      "epoch": 0.4471057884231537,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03139442205429077,
+      "kl": 0.0026619434356689453,
+      "learning_rate": 1.3863451256931286e-05,
+      "loss": -0.0061,
+      "num_tokens": 20189959.0,
+      "reward": 0.6022586822509766,
+      "reward_std": 0.06519456952810287,
+      "rewards/cross_solution_unittest/mean": 0.6022586226463318,
+      "rewards/cross_solution_unittest/std": 0.11483202874660492,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 49
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1825.0,
+      "completions/mean_length": 540.4635620117188,
+      "completions/mean_terminated_length": 532.5169677734375,
+      "completions/min_length": 243.0,
+      "completions/min_terminated_length": 243.0,
+      "entropy": 0.0771331787109375,
+      "epoch": 0.45623039635015683,
+      "frac_reward_zero_std": 0.0833333358168602,
+      "grad_norm": 0.03034847602248192,
+      "kl": 0.0031042098999023438,
+      "learning_rate": 1.356886221591872e-05,
+      "loss": 0.0005,
+      "num_tokens": 20614273.0,
+      "reward": 0.6334234476089478,
+      "reward_std": 0.05841737240552902,
+      "rewards/cross_solution_unittest/mean": 0.6334234476089478,
+      "rewards/cross_solution_unittest/std": 0.08473079651594162,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 50
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 2980.0,
+      "completions/max_terminated_length": 2980.0,
+      "completions/mean_length": 531.890625,
+      "completions/mean_terminated_length": 531.890625,
+      "completions/min_length": 178.0,
+      "completions/min_terminated_length": 178.0,
+      "entropy": 0.07479095458984375,
+      "epoch": 0.46535500427716,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03556211292743683,
+      "kl": 0.0032439231872558594,
+      "learning_rate": 1.3270679633174219e-05,
+      "loss": 0.0069,
+      "num_tokens": 21040351.0,
+      "reward": 0.6393711566925049,
+      "reward_std": 0.07185273617506027,
+      "rewards/cross_solution_unittest/mean": 0.6393711566925049,
+      "rewards/cross_solution_unittest/std": 0.11849500238895416,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 51
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1223.0,
+      "completions/mean_length": 527.3541870117188,
+      "completions/mean_terminated_length": 511.3507995605469,
+      "completions/min_length": 201.0,
+      "completions/min_terminated_length": 201.0,
+      "entropy": 0.0787811279296875,
+      "epoch": 0.4744796122041631,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.041180867701768875,
+      "kl": 0.0035219192504882812,
+      "learning_rate": 1.296920375328275e-05,
+      "loss": -0.0046,
+      "num_tokens": 21449467.0,
+      "reward": 0.6477403044700623,
+      "reward_std": 0.06604725122451782,
+      "rewards/cross_solution_unittest/mean": 0.6477403044700623,
+      "rewards/cross_solution_unittest/std": 0.12666402757167816,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 52
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 943.0,
+      "completions/mean_length": 505.08074951171875,
+      "completions/mean_terminated_length": 488.9607238769531,
+      "completions/min_length": 235.0,
+      "completions/min_terminated_length": 235.0,
+      "entropy": 0.079986572265625,
+      "epoch": 0.4836042201311662,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03628738969564438,
+      "kl": 0.004321575164794922,
+      "learning_rate": 1.266473813690035e-05,
+      "loss": 0.0045,
+      "num_tokens": 21858106.0,
+      "reward": 0.5700105428695679,
+      "reward_std": 0.06474246829748154,
+      "rewards/cross_solution_unittest/mean": 0.5700104832649231,
+      "rewards/cross_solution_unittest/std": 0.12813034653663635,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 53
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 975.0,
+      "completions/max_terminated_length": 975.0,
+      "completions/mean_length": 529.0443115234375,
+      "completions/mean_terminated_length": 529.0443115234375,
+      "completions/min_length": 253.0,
+      "completions/min_terminated_length": 253.0,
+      "entropy": 0.0794677734375,
+      "epoch": 0.49272882805816937,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.031926512718200684,
+      "kl": 0.003833293914794922,
+      "learning_rate": 1.2357589355094275e-05,
+      "loss": 0.0121,
+      "num_tokens": 22318391.0,
+      "reward": 0.6334779262542725,
+      "reward_std": 0.0830218717455864,
+      "rewards/cross_solution_unittest/mean": 0.6334779262542725,
+      "rewards/cross_solution_unittest/std": 0.1171019971370697,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 54
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1062.0,
+      "completions/max_terminated_length": 1062.0,
+      "completions/mean_length": 504.7161560058594,
+      "completions/mean_terminated_length": 504.7161560058594,
+      "completions/min_length": 267.0,
+      "completions/min_terminated_length": 267.0,
+      "entropy": 0.07977294921875,
+      "epoch": 0.5018534359851725,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03277221694588661,
+      "kl": 0.0038743019104003906,
+      "learning_rate": 1.2048066680651908e-05,
+      "loss": -0.005,
+      "num_tokens": 22699346.0,
+      "reward": 0.5762088298797607,
+      "reward_std": 0.0954534038901329,
+      "rewards/cross_solution_unittest/mean": 0.5762088298797607,
+      "rewards/cross_solution_unittest/std": 0.14616453647613525,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 55
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 2795.0,
+      "completions/mean_length": 505.390625,
+      "completions/mean_terminated_length": 497.35247802734375,
+      "completions/min_length": 256.0,
+      "completions/min_terminated_length": 256.0,
+      "entropy": 0.07952117919921875,
+      "epoch": 0.5109780439121756,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03929758444428444,
+      "kl": 0.005110740661621094,
+      "learning_rate": 1.1736481776669307e-05,
+      "loss": 0.003,
+      "num_tokens": 23081156.0,
+      "reward": 0.5718110799789429,
+      "reward_std": 0.08531081676483154,
+      "rewards/cross_solution_unittest/mean": 0.5718110203742981,
+      "rewards/cross_solution_unittest/std": 0.12530149519443512,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 56
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1219.0,
+      "completions/mean_length": 516.2057495117188,
+      "completions/mean_terminated_length": 508.1958312988281,
+      "completions/min_length": 280.0,
+      "completions/min_terminated_length": 280.0,
+      "entropy": 0.0814666748046875,
+      "epoch": 0.5201026518391788,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0367487296462059,
+      "kl": 0.004551410675048828,
+      "learning_rate": 1.1423148382732854e-05,
+      "loss": 0.0163,
+      "num_tokens": 23529571.0,
+      "reward": 0.6225696802139282,
+      "reward_std": 0.060477182269096375,
+      "rewards/cross_solution_unittest/mean": 0.6225696802139282,
+      "rewards/cross_solution_unittest/std": 0.1196565330028534,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 57
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.953125,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1001.0,
+      "completions/mean_length": 507.19793701171875,
+      "completions/mean_terminated_length": 482.97113037109375,
+      "completions/min_length": 219.0,
+      "completions/min_terminated_length": 219.0,
+      "entropy": 0.088714599609375,
+      "epoch": 0.5292272597661819,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.036932192742824554,
+      "kl": 0.005054950714111328,
+      "learning_rate": 1.1108381999010111e-05,
+      "loss": -0.0237,
+      "num_tokens": 23952099.0,
+      "reward": 0.61222243309021,
+      "reward_std": 0.07403381913900375,
+      "rewards/cross_solution_unittest/mean": 0.61222243309021,
+      "rewards/cross_solution_unittest/std": 0.12056523561477661,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 58
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.9375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 3118.0,
+      "completions/mean_length": 512.6849365234375,
+      "completions/mean_terminated_length": 480.35528564453125,
+      "completions/min_length": 241.0,
+      "completions/min_terminated_length": 241.0,
+      "entropy": 0.0914764404296875,
+      "epoch": 0.5383518676931851,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03964272886514664,
+      "kl": 0.006877422332763672,
+      "learning_rate": 1.0792499568567885e-05,
+      "loss": 0.0078,
+      "num_tokens": 24370834.0,
+      "reward": 0.6122474074363708,
+      "reward_std": 0.062438443303108215,
+      "rewards/cross_solution_unittest/mean": 0.6122473478317261,
+      "rewards/cross_solution_unittest/std": 0.11854556947946548,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 59
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1433.0,
+      "completions/max_terminated_length": 1433.0,
+      "completions/mean_length": 510.22918701171875,
+      "completions/mean_terminated_length": 510.22918701171875,
+      "completions/min_length": 283.0,
+      "completions/min_terminated_length": 283.0,
+      "entropy": 0.0788726806640625,
+      "epoch": 0.5474764756201882,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.037243351340293884,
+      "kl": 0.005549430847167969,
+      "learning_rate": 1.0475819158237426e-05,
+      "loss": 0.0003,
+      "num_tokens": 24808546.0,
+      "reward": 0.6103326082229614,
+      "reward_std": 0.0680248886346817,
+      "rewards/cross_solution_unittest/mean": 0.6103325486183167,
+      "rewards/cross_solution_unittest/std": 0.11332948505878448,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 60
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1041.0,
+      "completions/max_terminated_length": 1041.0,
+      "completions/mean_length": 544.8724365234375,
+      "completions/mean_terminated_length": 544.8724365234375,
+      "completions/min_length": 274.0,
+      "completions/min_terminated_length": 274.0,
+      "entropy": 0.07517242431640625,
+      "epoch": 0.5566010835471913,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03436488285660744,
+      "kl": 0.004729270935058594,
+      "learning_rate": 1.015865963834808e-05,
+      "loss": 0.0028,
+      "num_tokens": 25265381.0,
+      "reward": 0.5960478782653809,
+      "reward_std": 0.07282952964305878,
+      "rewards/cross_solution_unittest/mean": 0.5960478782653809,
+      "rewards/cross_solution_unittest/std": 0.12416920810937881,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 61
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1118.0,
+      "completions/max_terminated_length": 1118.0,
+      "completions/mean_length": 525.9349365234375,
+      "completions/mean_terminated_length": 525.9349365234375,
+      "completions/min_length": 271.0,
+      "completions/min_terminated_length": 271.0,
+      "entropy": 0.0847625732421875,
+      "epoch": 0.5657256914741945,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03560968488454819,
+      "kl": 0.005660057067871094,
+      "learning_rate": 9.841340361651921e-06,
+      "loss": 0.0144,
+      "num_tokens": 25690764.0,
+      "reward": 0.6058902144432068,
+      "reward_std": 0.0673966109752655,
+      "rewards/cross_solution_unittest/mean": 0.6058902144432068,
+      "rewards/cross_solution_unittest/std": 0.0988718718290329,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 62
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.953125,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 972.0,
+      "completions/mean_length": 502.2083435058594,
+      "completions/mean_terminated_length": 477.9422607421875,
+      "completions/min_length": 223.0,
+      "completions/min_terminated_length": 223.0,
+      "entropy": 0.08184814453125,
+      "epoch": 0.5748502994011976,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.037654757499694824,
+      "kl": 0.0056934356689453125,
+      "learning_rate": 9.524180841762577e-06,
+      "loss": 0.0073,
+      "num_tokens": 26075548.0,
+      "reward": 0.6195604801177979,
+      "reward_std": 0.09131857007741928,
+      "rewards/cross_solution_unittest/mean": 0.6195604205131531,
+      "rewards/cross_solution_unittest/std": 0.12209580838680267,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 63
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1050.0,
+      "completions/max_terminated_length": 1050.0,
+      "completions/mean_length": 504.7395935058594,
+      "completions/mean_terminated_length": 504.7395935058594,
+      "completions/min_length": 280.0,
+      "completions/min_terminated_length": 280.0,
+      "entropy": 0.077972412109375,
+      "epoch": 0.5839749073282008,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04175296798348427,
+      "kl": 0.005429267883300781,
+      "learning_rate": 9.207500431432115e-06,
+      "loss": -0.0079,
+      "num_tokens": 26471072.0,
+      "reward": 0.593117356300354,
+      "reward_std": 0.08295892179012299,
+      "rewards/cross_solution_unittest/mean": 0.5931172966957092,
+      "rewards/cross_solution_unittest/std": 0.1298408806324005,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 64
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 849.0,
+      "completions/max_terminated_length": 849.0,
+      "completions/mean_length": 464.1145935058594,
+      "completions/mean_terminated_length": 464.1145935058594,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
+      "entropy": 0.089385986328125,
+      "epoch": 0.5930995152552039,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.039406850934028625,
+      "kl": 0.006228446960449219,
+      "learning_rate": 8.89161800098989e-06,
+      "loss": -0.0001,
+      "num_tokens": 26854340.0,
+      "reward": 0.6415243148803711,
+      "reward_std": 0.0817657858133316,
+      "rewards/cross_solution_unittest/mean": 0.6415242552757263,
+      "rewards/cross_solution_unittest/std": 0.13109222054481506,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 65
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 2036.0,
+      "completions/mean_length": 537.296875,
+      "completions/mean_terminated_length": 521.3455810546875,
+      "completions/min_length": 241.0,
+      "completions/min_terminated_length": 241.0,
+      "entropy": 0.082550048828125,
+      "epoch": 0.602224123182207,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.034542761743068695,
+      "kl": 0.005867958068847656,
+      "learning_rate": 8.576851617267151e-06,
+      "loss": -0.0014,
+      "num_tokens": 27332642.0,
+      "reward": 0.6036405563354492,
+      "reward_std": 0.0659734234213829,
+      "rewards/cross_solution_unittest/mean": 0.6036404967308044,
+      "rewards/cross_solution_unittest/std": 0.10401112586259842,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 66
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 956.0,
+      "completions/mean_length": 511.71875,
+      "completions/mean_terminated_length": 495.6335144042969,
+      "completions/min_length": 235.0,
+      "completions/min_terminated_length": 235.0,
+      "entropy": 0.0772247314453125,
+      "epoch": 0.6113487311092102,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03523527458310127,
+      "kl": 0.0067005157470703125,
+      "learning_rate": 8.263518223330698e-06,
+      "loss": 0.0031,
+      "num_tokens": 27795114.0,
+      "reward": 0.6518998146057129,
+      "reward_std": 0.06018558144569397,
+      "rewards/cross_solution_unittest/mean": 0.6518998146057129,
+      "rewards/cross_solution_unittest/std": 0.0940474197268486,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 67
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1068.0,
+      "completions/mean_length": 534.8724365234375,
+      "completions/mean_terminated_length": 526.9112548828125,
+      "completions/min_length": 280.0,
+      "completions/min_terminated_length": 280.0,
+      "entropy": 0.0787353515625,
+      "epoch": 0.6204733390362133,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.031150035560131073,
+      "kl": 0.005561351776123047,
+      "learning_rate": 7.951933319348095e-06,
+      "loss": -0.0163,
+      "num_tokens": 28203505.0,
+      "reward": 0.6172535419464111,
+      "reward_std": 0.07265746593475342,
+      "rewards/cross_solution_unittest/mean": 0.6172534823417664,
+      "rewards/cross_solution_unittest/std": 0.10251819342374802,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 68
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 2633.0,
+      "completions/max_terminated_length": 2633.0,
+      "completions/mean_length": 504.0520935058594,
+      "completions/mean_terminated_length": 504.0520935058594,
+      "completions/min_length": 195.0,
+      "completions/min_terminated_length": 195.0,
+      "entropy": 0.0829925537109375,
+      "epoch": 0.6295979469632165,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03765825554728508,
+      "kl": 0.007781982421875,
+      "learning_rate": 7.642410644905726e-06,
+      "loss": -0.0102,
+      "num_tokens": 28643677.0,
+      "reward": 0.5815111398696899,
+      "reward_std": 0.07605060935020447,
+      "rewards/cross_solution_unittest/mean": 0.5815110802650452,
+      "rewards/cross_solution_unittest/std": 0.1344773769378662,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 69
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1305.0,
+      "completions/max_terminated_length": 1305.0,
+      "completions/mean_length": 521.3515625,
+      "completions/mean_terminated_length": 521.3515625,
+      "completions/min_length": 235.0,
+      "completions/min_terminated_length": 235.0,
+      "entropy": 0.076080322265625,
+      "epoch": 0.6387225548902196,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03361423686146736,
+      "kl": 0.005164623260498047,
+      "learning_rate": 7.335261863099652e-06,
+      "loss": -0.0013,
+      "num_tokens": 29087396.0,
+      "reward": 0.5668891668319702,
+      "reward_std": 0.0703488141298294,
+      "rewards/cross_solution_unittest/mean": 0.5668891072273254,
+      "rewards/cross_solution_unittest/std": 0.1609523743391037,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 70
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.9375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1279.0,
+      "completions/mean_length": 543.7734375,
+      "completions/mean_terminated_length": 511.7710876464844,
+      "completions/min_length": 283.0,
+      "completions/min_terminated_length": 283.0,
+      "entropy": 0.08349609375,
+      "epoch": 0.6478471628172227,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03806523233652115,
+      "kl": 0.006535530090332031,
+      "learning_rate": 7.0307962467172555e-06,
+      "loss": 0.009,
+      "num_tokens": 29509857.0,
+      "reward": 0.5793834924697876,
+      "reward_std": 0.10175223648548126,
+      "rewards/cross_solution_unittest/mean": 0.5793834328651428,
+      "rewards/cross_solution_unittest/std": 0.13090623915195465,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 71
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1215.0,
+      "completions/mean_length": 522.6796875,
+      "completions/mean_terminated_length": 514.6867065429688,
+      "completions/min_length": 247.0,
+      "completions/min_terminated_length": 247.0,
+      "entropy": 0.08115386962890625,
+      "epoch": 0.6569717707442259,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.038980960845947266,
+      "kl": 0.007760047912597656,
+      "learning_rate": 6.729320366825785e-06,
+      "loss": 0.0078,
+      "num_tokens": 29920282.0,
+      "reward": 0.623957097530365,
+      "reward_std": 0.05566735565662384,
+      "rewards/cross_solution_unittest/mean": 0.6239569783210754,
+      "rewards/cross_solution_unittest/std": 0.08572336286306381,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 72
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1231.0,
+      "completions/mean_length": 501.3255310058594,
+      "completions/mean_terminated_length": 485.1858825683594,
+      "completions/min_length": 202.0,
+      "completions/min_terminated_length": 202.0,
+      "entropy": 0.0852813720703125,
+      "epoch": 0.666096378671229,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.040267642587423325,
+      "kl": 0.007144451141357422,
+      "learning_rate": 6.431137784081283e-06,
+      "loss": -0.0006,
+      "num_tokens": 30330855.0,
+      "reward": 0.6102144122123718,
+      "reward_std": 0.06656725704669952,
+      "rewards/cross_solution_unittest/mean": 0.6102144122123718,
+      "rewards/cross_solution_unittest/std": 0.10697145015001297,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 73
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1347.0,
+      "completions/max_terminated_length": 1347.0,
+      "completions/mean_length": 525.5677490234375,
+      "completions/mean_terminated_length": 525.5677490234375,
+      "completions/min_length": 206.0,
+      "completions/min_terminated_length": 206.0,
+      "entropy": 0.0807037353515625,
+      "epoch": 0.6752209865982322,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03388255834579468,
+      "kl": 0.007240772247314453,
+      "learning_rate": 6.136548743068713e-06,
+      "loss": 0.0082,
+      "num_tokens": 30740081.0,
+      "reward": 0.6364445686340332,
+      "reward_std": 0.06512817740440369,
+      "rewards/cross_solution_unittest/mean": 0.6364445686340332,
+      "rewards/cross_solution_unittest/std": 0.13945284485816956,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 74
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1336.0,
+      "completions/max_terminated_length": 1336.0,
+      "completions/mean_length": 518.7994995117188,
+      "completions/mean_terminated_length": 518.7994995117188,
+      "completions/min_length": 272.0,
+      "completions/min_terminated_length": 272.0,
+      "entropy": 0.091766357421875,
+      "epoch": 0.6843455945252352,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03697756305336952,
+      "kl": 0.006485462188720703,
+      "learning_rate": 5.845849869981137e-06,
+      "loss": -0.0074,
+      "num_tokens": 31119772.0,
+      "reward": 0.5954554080963135,
+      "reward_std": 0.0874066948890686,
+      "rewards/cross_solution_unittest/mean": 0.5954554677009583,
+      "rewards/cross_solution_unittest/std": 0.10502495616674423,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 75
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 992.0,
+      "completions/mean_length": 470.27606201171875,
+      "completions/mean_terminated_length": 462.1462097167969,
+      "completions/min_length": 239.0,
+      "completions/min_terminated_length": 239.0,
+      "entropy": 0.09112548828125,
+      "epoch": 0.6934702024522383,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03978769853711128,
+      "kl": 0.008324623107910156,
+      "learning_rate": 5.559333873942259e-06,
+      "loss": 0.0145,
+      "num_tokens": 31463730.0,
+      "reward": 0.6490760445594788,
+      "reward_std": 0.06858085095882416,
+      "rewards/cross_solution_unittest/mean": 0.6490760445594788,
+      "rewards/cross_solution_unittest/std": 0.1581326723098755,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 76
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1465.0,
+      "completions/mean_length": 508.54949951171875,
+      "completions/mean_terminated_length": 500.51959228515625,
+      "completions/min_length": 242.0,
+      "completions/min_terminated_length": 242.0,
+      "entropy": 0.0853118896484375,
+      "epoch": 0.7025948103792415,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03948879987001419,
+      "kl": 0.00698089599609375,
+      "learning_rate": 5.277289252273175e-06,
+      "loss": 0.0121,
+      "num_tokens": 31872545.0,
+      "reward": 0.5989640951156616,
+      "reward_std": 0.10313881933689117,
+      "rewards/cross_solution_unittest/mean": 0.5989640355110168,
+      "rewards/cross_solution_unittest/std": 0.14925776422023773,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 77
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 2062.0,
+      "completions/mean_length": 547.5208740234375,
+      "completions/mean_terminated_length": 539.5927124023438,
+      "completions/min_length": 237.0,
+      "completions/min_terminated_length": 237.0,
+      "entropy": 0.0765838623046875,
+      "epoch": 0.7117194183062446,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04029017686843872,
+      "kl": 0.00771331787109375,
+      "learning_rate": 5.000000000000003e-06,
+      "loss": -0.0006,
+      "num_tokens": 32327125.0,
+      "reward": 0.6118708848953247,
+      "reward_std": 0.08621498942375183,
+      "rewards/cross_solution_unittest/mean": 0.6118709444999695,
+      "rewards/cross_solution_unittest/std": 0.146837517619133,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 78
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1261.0,
+      "completions/mean_length": 568.5625,
+      "completions/mean_terminated_length": 504.40423583984375,
+      "completions/min_length": 235.0,
+      "completions/min_terminated_length": 235.0,
+      "entropy": 0.0814666748046875,
+      "epoch": 0.7208440262332478,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03578219935297966,
+      "kl": 0.007190704345703125,
+      "learning_rate": 4.727745323894976e-06,
+      "loss": -0.0134,
+      "num_tokens": 32743281.0,
+      "reward": 0.6152719855308533,
+      "reward_std": 0.061369992792606354,
+      "rewards/cross_solution_unittest/mean": 0.6152719855308533,
+      "rewards/cross_solution_unittest/std": 0.12953531742095947,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 79
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1041.0,
+      "completions/mean_length": 498.3177185058594,
+      "completions/mean_terminated_length": 490.2611083984375,
+      "completions/min_length": 226.0,
+      "completions/min_terminated_length": 226.0,
+      "entropy": 0.0795440673828125,
+      "epoch": 0.7299686341602509,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03637859597802162,
+      "kl": 0.00676727294921875,
+      "learning_rate": 4.460799361338898e-06,
+      "loss": -0.0083,
+      "num_tokens": 33113451.0,
+      "reward": 0.6233752369880676,
+      "reward_std": 0.0716649666428566,
+      "rewards/cross_solution_unittest/mean": 0.6233752369880676,
+      "rewards/cross_solution_unittest/std": 0.14779163897037506,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 80
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 2090.0,
+      "completions/mean_length": 507.8724060058594,
+      "completions/mean_terminated_length": 499.8407287597656,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
+      "entropy": 0.08404541015625,
+      "epoch": 0.739093242087254,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.034254688769578934,
+      "kl": 0.006606101989746094,
+      "learning_rate": 4.19943090428802e-06,
+      "loss": 0.0015,
+      "num_tokens": 33504634.0,
+      "reward": 0.5900766849517822,
+      "reward_std": 0.07616183161735535,
+      "rewards/cross_solution_unittest/mean": 0.5900766253471375,
+      "rewards/cross_solution_unittest/std": 0.15281453728675842,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 81
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1011.0,
+      "completions/mean_length": 493.8177185058594,
+      "completions/mean_terminated_length": 485.7493591308594,
+      "completions/min_length": 226.0,
+      "completions/min_terminated_length": 226.0,
+      "entropy": 0.08441162109375,
+      "epoch": 0.7482178500142572,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03650706261396408,
+      "kl": 0.007792472839355469,
+      "learning_rate": 3.943903128623336e-06,
+      "loss": -0.0018,
+      "num_tokens": 33896656.0,
+      "reward": 0.6140320301055908,
+      "reward_std": 0.07920640707015991,
+      "rewards/cross_solution_unittest/mean": 0.614031970500946,
+      "rewards/cross_solution_unittest/std": 0.12956902384757996,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 82
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 866.0,
+      "completions/mean_length": 524.1484375,
+      "completions/mean_terminated_length": 508.1282653808594,
+      "completions/min_length": 265.0,
+      "completions/min_terminated_length": 265.0,
+      "entropy": 0.081451416015625,
+      "epoch": 0.7573424579412603,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.034576863050460815,
+      "kl": 0.007690906524658203,
+      "learning_rate": 3.6944733291547784e-06,
+      "loss": -0.0122,
+      "num_tokens": 34306081.0,
+      "reward": 0.6044012308120728,
+      "reward_std": 0.07356821745634079,
+      "rewards/cross_solution_unittest/mean": 0.6044012904167175,
+      "rewards/cross_solution_unittest/std": 0.12973514199256897,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 83
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1150.0,
+      "completions/mean_length": 497.54949951171875,
+      "completions/mean_terminated_length": 489.4908752441406,
+      "completions/min_length": 222.0,
+      "completions/min_terminated_length": 222.0,
+      "entropy": 0.087127685546875,
+      "epoch": 0.7664670658682635,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04500949755311012,
+      "kl": 0.008170127868652344,
+      "learning_rate": 3.4513926605471504e-06,
+      "loss": -0.0097,
+      "num_tokens": 34654848.0,
+      "reward": 0.563988208770752,
+      "reward_std": 0.09649153798818588,
+      "rewards/cross_solution_unittest/mean": 0.563988208770752,
+      "rewards/cross_solution_unittest/std": 0.17347638309001923,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 84
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1627.0,
+      "completions/mean_length": 516.9948120117188,
+      "completions/mean_terminated_length": 508.9869384765625,
+      "completions/min_length": 236.0,
+      "completions/min_terminated_length": 236.0,
+      "entropy": 0.0800933837890625,
+      "epoch": 0.7755916737952666,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03465050458908081,
+      "kl": 0.007006645202636719,
+      "learning_rate": 3.2149058844286796e-06,
+      "loss": 0.0084,
+      "num_tokens": 35092086.0,
+      "reward": 0.577928900718689,
+      "reward_std": 0.08402758836746216,
+      "rewards/cross_solution_unittest/mean": 0.5779288411140442,
+      "rewards/cross_solution_unittest/std": 0.1539035141468048,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 85
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 905.0,
+      "completions/mean_length": 493.29168701171875,
+      "completions/mean_terminated_length": 477.1099548339844,
+      "completions/min_length": 250.0,
+      "completions/min_terminated_length": 250.0,
+      "entropy": 0.0883026123046875,
+      "epoch": 0.7847162817222697,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.040905412286520004,
+      "kl": 0.008014678955078125,
+      "learning_rate": 2.9852511229367862e-06,
+      "loss": -0.0056,
+      "num_tokens": 35510442.0,
+      "reward": 0.6279939413070679,
+      "reward_std": 0.08435385674238205,
+      "rewards/cross_solution_unittest/mean": 0.6279939413070679,
+      "rewards/cross_solution_unittest/std": 0.13109418749809265,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 86
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 2231.0,
+      "completions/mean_length": 529.7109375,
+      "completions/mean_terminated_length": 513.7199096679688,
+      "completions/min_length": 266.0,
+      "completions/min_terminated_length": 266.0,
+      "entropy": 0.085418701171875,
+      "epoch": 0.7938408896492729,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04118010401725769,
+      "kl": 0.007990837097167969,
+      "learning_rate": 2.7626596189492983e-06,
+      "loss": -0.0104,
+      "num_tokens": 35943883.0,
+      "reward": 0.6178367137908936,
+      "reward_std": 0.07810844480991364,
+      "rewards/cross_solution_unittest/mean": 0.6178367137908936,
+      "rewards/cross_solution_unittest/std": 0.12682530283927917,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 87
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 866.0,
+      "completions/max_terminated_length": 866.0,
+      "completions/mean_length": 436.6796875,
+      "completions/mean_terminated_length": 436.6796875,
+      "completions/min_length": 217.0,
+      "completions/min_terminated_length": 217.0,
+      "entropy": 0.0947723388671875,
+      "epoch": 0.802965497576276,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.05317376181483269,
+      "kl": 0.009276390075683594,
+      "learning_rate": 2.5473555032424534e-06,
+      "loss": 0.0077,
+      "num_tokens": 36297980.0,
+      "reward": 0.6129441261291504,
+      "reward_std": 0.08101849257946014,
+      "rewards/cross_solution_unittest/mean": 0.6129441261291504,
+      "rewards/cross_solution_unittest/std": 0.15165738761425018,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 88
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 991.0,
+      "completions/max_terminated_length": 991.0,
+      "completions/mean_length": 515.4791870117188,
+      "completions/mean_terminated_length": 515.4791870117188,
+      "completions/min_length": 264.0,
+      "completions/min_terminated_length": 264.0,
+      "entropy": 0.07867431640625,
+      "epoch": 0.8120901055032792,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03746452555060387,
+      "kl": 0.0072307586669921875,
+      "learning_rate": 2.339555568810221e-06,
+      "loss": 0.014,
+      "num_tokens": 36730140.0,
+      "reward": 0.5748163461685181,
+      "reward_std": 0.06574384868144989,
+      "rewards/cross_solution_unittest/mean": 0.5748162865638733,
+      "rewards/cross_solution_unittest/std": 0.12489957362413406,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 89
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1047.0,
+      "completions/mean_length": 518.2213745117188,
+      "completions/mean_terminated_length": 502.170166015625,
+      "completions/min_length": 280.0,
+      "completions/min_terminated_length": 280.0,
+      "entropy": 0.07883453369140625,
+      "epoch": 0.8212147134302823,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03742819279432297,
+      "kl": 0.007193565368652344,
+      "learning_rate": 2.1394690525721275e-06,
+      "loss": -0.0006,
+      "num_tokens": 37153593.0,
+      "reward": 0.5915634632110596,
+      "reward_std": 0.07696962356567383,
+      "rewards/cross_solution_unittest/mean": 0.5915634632110596,
+      "rewards/cross_solution_unittest/std": 0.12713728845119476,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 90
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 922.0,
+      "completions/mean_length": 477.1171875,
+      "completions/mean_terminated_length": 469.0052185058594,
+      "completions/min_length": 235.0,
+      "completions/min_terminated_length": 235.0,
+      "entropy": 0.0898284912109375,
+      "epoch": 0.8303393213572854,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04124828428030014,
+      "kl": 0.010035514831542969,
+      "learning_rate": 1.947297424689414e-06,
+      "loss": -0.0044,
+      "num_tokens": 37537306.0,
+      "reward": 0.6299624443054199,
+      "reward_std": 0.08996909856796265,
+      "rewards/cross_solution_unittest/mean": 0.6299623847007751,
+      "rewards/cross_solution_unittest/std": 0.1566157191991806,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 91
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1312.0,
+      "completions/mean_length": 511.70574951171875,
+      "completions/mean_terminated_length": 495.62042236328125,
+      "completions/min_length": 205.0,
+      "completions/min_terminated_length": 205.0,
+      "entropy": 0.0867462158203125,
+      "epoch": 0.8394639292842886,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04116717353463173,
+      "kl": 0.009945869445800781,
+      "learning_rate": 1.7632341857016733e-06,
+      "loss": 0.0106,
+      "num_tokens": 37959049.0,
+      "reward": 0.613093376159668,
+      "reward_std": 0.07671508938074112,
+      "rewards/cross_solution_unittest/mean": 0.6130933165550232,
+      "rewards/cross_solution_unittest/std": 0.09333295375108719,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 92
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.921875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1230.0,
+      "completions/mean_length": 576.078125,
+      "completions/mean_terminated_length": 536.3958129882812,
+      "completions/min_length": 11.0,
+      "completions/min_terminated_length": 11.0,
+      "entropy": 0.07297515869140625,
+      "epoch": 0.8485885372112917,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03428766503930092,
+      "kl": 0.007388114929199219,
+      "learning_rate": 1.587464671688187e-06,
+      "loss": -0.0183,
+      "num_tokens": 38399059.0,
+      "reward": 0.618889331817627,
+      "reward_std": 0.08409196138381958,
+      "rewards/cross_solution_unittest/mean": 0.618889331817627,
+      "rewards/cross_solution_unittest/std": 0.12098271399736404,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 93
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1236.0,
+      "completions/mean_length": 530.6068115234375,
+      "completions/mean_terminated_length": 522.6344604492188,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
+      "entropy": 0.07482147216796875,
+      "epoch": 0.8577131451382949,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0353349931538105,
+      "kl": 0.006430149078369141,
+      "learning_rate": 1.4201658676502294e-06,
+      "loss": 0.0119,
+      "num_tokens": 38818392.0,
+      "reward": 0.658619225025177,
+      "reward_std": 0.08192513883113861,
+      "rewards/cross_solution_unittest/mean": 0.6586191654205322,
+      "rewards/cross_solution_unittest/std": 0.11717738956212997,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 94
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 929.0,
+      "completions/max_terminated_length": 929.0,
+      "completions/mean_length": 493.29168701171875,
+      "completions/mean_terminated_length": 493.29168701171875,
+      "completions/min_length": 234.0,
+      "completions/min_terminated_length": 234.0,
+      "entropy": 0.0816497802734375,
+      "epoch": 0.866837753065298,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.0350792221724987,
+      "kl": 0.007628440856933594,
+      "learning_rate": 1.2615062293021508e-06,
+      "loss": 0.002,
+      "num_tokens": 39264744.0,
+      "reward": 0.6107174158096313,
+      "reward_std": 0.06959541141986847,
+      "rewards/cross_solution_unittest/mean": 0.6107173562049866,
+      "rewards/cross_solution_unittest/std": 0.09727025777101517,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 95
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 1227.0,
+      "completions/mean_length": 532.9869995117188,
+      "completions/mean_terminated_length": 525.0208740234375,
+      "completions/min_length": 272.0,
+      "completions/min_terminated_length": 272.0,
+      "entropy": 0.079864501953125,
+      "epoch": 0.8759623609923011,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03818963095545769,
+      "kl": 0.006740570068359375,
+      "learning_rate": 1.1116455134507665e-06,
+      "loss": -0.0028,
+      "num_tokens": 39659039.0,
+      "reward": 0.5914241671562195,
+      "reward_std": 0.07413486391305923,
+      "rewards/cross_solution_unittest/mean": 0.5914241671562195,
+      "rewards/cross_solution_unittest/std": 0.14320488274097443,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 96
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1008.0,
+      "completions/max_terminated_length": 1008.0,
+      "completions/mean_length": 460.04168701171875,
+      "completions/mean_terminated_length": 460.04168701171875,
+      "completions/min_length": 205.0,
+      "completions/min_terminated_length": 205.0,
+      "entropy": 0.0875091552734375,
+      "epoch": 0.8850869689193043,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.041273754090070724,
+      "kl": 0.007297515869140625,
+      "learning_rate": 9.707346171337895e-07,
+      "loss": -0.0054,
+      "num_tokens": 39981139.0,
+      "reward": 0.6115624308586121,
+      "reward_std": 0.07325311750173569,
+      "rewards/cross_solution_unittest/mean": 0.6115624308586121,
+      "rewards/cross_solution_unittest/std": 0.09799737483263016,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 97
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 1656.0,
+      "completions/max_terminated_length": 1656.0,
+      "completions/mean_length": 526.7291870117188,
+      "completions/mean_terminated_length": 526.7291870117188,
+      "completions/min_length": 272.0,
+      "completions/min_terminated_length": 272.0,
+      "entropy": 0.0795745849609375,
+      "epoch": 0.8942115768463074,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03502219542860985,
+      "kl": 0.006832599639892578,
+      "learning_rate": 8.389154256793042e-07,
+      "loss": -0.0011,
+      "num_tokens": 40425327.0,
+      "reward": 0.6242746710777283,
+      "reward_std": 0.06769298017024994,
+      "rewards/cross_solution_unittest/mean": 0.6242746114730835,
+      "rewards/cross_solution_unittest/std": 0.11656895279884338,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 98
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 2244.0,
+      "completions/mean_length": 509.94793701171875,
+      "completions/mean_terminated_length": 501.92169189453125,
+      "completions/min_length": 226.0,
+      "completions/min_terminated_length": 226.0,
+      "entropy": 0.0839080810546875,
+      "epoch": 0.9033361847733106,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03684497997164726,
+      "kl": 0.009736061096191406,
+      "learning_rate": 7.163206698392744e-07,
+      "loss": 0.0016,
+      "num_tokens": 40804411.0,
+      "reward": 0.5832025408744812,
+      "reward_std": 0.08364126086235046,
+      "rewards/cross_solution_unittest/mean": 0.5832025408744812,
+      "rewards/cross_solution_unittest/std": 0.15208092331886292,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 99
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 859.0,
+      "completions/max_terminated_length": 859.0,
+      "completions/mean_length": 454.2734375,
+      "completions/mean_terminated_length": 454.2734375,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
+      "entropy": 0.094268798828125,
+      "epoch": 0.9124607927003137,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03873429447412491,
+      "kl": 0.009373664855957031,
+      "learning_rate": 6.030737921409169e-07,
+      "loss": 0.0099,
+      "num_tokens": 41217760.0,
+      "reward": 0.6278964281082153,
+      "reward_std": 0.07669099420309067,
+      "rewards/cross_solution_unittest/mean": 0.6278963685035706,
+      "rewards/cross_solution_unittest/std": 0.09835278242826462,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 100
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 919.0,
+      "completions/max_terminated_length": 919.0,
+      "completions/mean_length": 482.8020935058594,
+      "completions/mean_terminated_length": 482.8020935058594,
+      "completions/min_length": 244.0,
+      "completions/min_terminated_length": 244.0,
+      "entropy": 0.08026123046875,
+      "epoch": 0.9215854006273168,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03827023506164551,
+      "kl": 0.007940292358398438,
+      "learning_rate": 4.992888225905467e-07,
+      "loss": 0.0048,
+      "num_tokens": 41606180.0,
+      "reward": 0.589016854763031,
+      "reward_std": 0.10000002384185791,
+      "rewards/cross_solution_unittest/mean": 0.589016854763031,
+      "rewards/cross_solution_unittest/std": 0.1567855030298233,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 101
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 947.0,
+      "completions/max_terminated_length": 947.0,
+      "completions/mean_length": 507.6770935058594,
+      "completions/mean_terminated_length": 507.6770935058594,
+      "completions/min_length": 176.0,
+      "completions/min_terminated_length": 176.0,
+      "entropy": 0.0762786865234375,
+      "epoch": 0.93071000855432,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03409411013126373,
+      "kl": 0.007808685302734375,
+      "learning_rate": 4.0507026385502747e-07,
+      "loss": -0.0019,
+      "num_tokens": 42037104.0,
+      "reward": 0.6218315958976746,
+      "reward_std": 0.06061244010925293,
+      "rewards/cross_solution_unittest/mean": 0.6218315362930298,
+      "rewards/cross_solution_unittest/std": 0.0926637127995491,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 102
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 2635.0,
+      "completions/max_terminated_length": 2635.0,
+      "completions/mean_length": 476.1302185058594,
+      "completions/mean_terminated_length": 476.1302185058594,
+      "completions/min_length": 205.0,
+      "completions/min_terminated_length": 205.0,
+      "entropy": 0.0820465087890625,
+      "epoch": 0.939834616481323,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.042139455676078796,
+      "kl": 0.009988784790039062,
+      "learning_rate": 3.2051298603643754e-07,
+      "loss": 0.0065,
+      "num_tokens": 42407794.0,
+      "reward": 0.6024968028068542,
+      "reward_std": 0.06899292767047882,
+      "rewards/cross_solution_unittest/mean": 0.6024968028068542,
+      "rewards/cross_solution_unittest/std": 0.09149476885795593,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 103
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 913.0,
+      "completions/mean_length": 498.86199951171875,
+      "completions/mean_terminated_length": 490.8067932128906,
+      "completions/min_length": 267.0,
+      "completions/min_terminated_length": 267.0,
+      "entropy": 0.0809326171875,
+      "epoch": 0.9489592244083263,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04004906862974167,
+      "kl": 0.008276939392089844,
+      "learning_rate": 2.4570213114592957e-07,
+      "loss": -0.0043,
+      "num_tokens": 42825253.0,
+      "reward": 0.6223401427268982,
+      "reward_std": 0.06938310712575912,
+      "rewards/cross_solution_unittest/mean": 0.622340202331543,
+      "rewards/cross_solution_unittest/std": 0.10645981132984161,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 104
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 3572.0,
+      "completions/max_terminated_length": 3572.0,
+      "completions/mean_length": 478.1145935058594,
+      "completions/mean_terminated_length": 478.1145935058594,
+      "completions/min_length": 230.0,
+      "completions/min_terminated_length": 230.0,
+      "entropy": 0.08770751953125,
+      "epoch": 0.9580838323353293,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.04208485782146454,
+      "kl": 0.01025390625,
+      "learning_rate": 1.8071302737293294e-07,
+      "loss": 0.0031,
+      "num_tokens": 43193629.0,
+      "reward": 0.6199694871902466,
+      "reward_std": 0.08685685694217682,
+      "rewards/cross_solution_unittest/mean": 0.6199694275856018,
+      "rewards/cross_solution_unittest/std": 0.14252996444702148,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 105
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 798.0,
+      "completions/mean_length": 528.5625,
+      "completions/mean_terminated_length": 512.5654296875,
+      "completions/min_length": 286.0,
+      "completions/min_terminated_length": 286.0,
+      "entropy": 0.07763671875,
+      "epoch": 0.9672084402623324,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03514657914638519,
+      "kl": 0.007828712463378906,
+      "learning_rate": 1.2561111323605714e-07,
+      "loss": 0.0011,
+      "num_tokens": 43621285.0,
+      "reward": 0.6022021770477295,
+      "reward_std": 0.06825901567935944,
+      "rewards/cross_solution_unittest/mean": 0.6022021770477295,
+      "rewards/cross_solution_unittest/std": 0.11643550544977188,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 106
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 3085.0,
+      "completions/mean_length": 521.5130615234375,
+      "completions/mean_terminated_length": 513.5169677734375,
+      "completions/min_length": 250.0,
+      "completions/min_terminated_length": 250.0,
+      "entropy": 0.0900115966796875,
+      "epoch": 0.9763330481893356,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03944119065999985,
+      "kl": 0.007457733154296875,
+      "learning_rate": 8.04518716920466e-08,
+      "loss": 0.0072,
+      "num_tokens": 44035422.0,
+      "reward": 0.6426176428794861,
+      "reward_std": 0.05617108941078186,
+      "rewards/cross_solution_unittest/mean": 0.6426176428794861,
+      "rewards/cross_solution_unittest/std": 0.09470611810684204,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 107
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -4.96875,
+      "completions/max_length": 3584.0,
+      "completions/max_terminated_length": 938.0,
+      "completions/mean_length": 545.078125,
+      "completions/mean_terminated_length": 529.1675415039062,
+      "completions/min_length": 240.0,
+      "completions/min_terminated_length": 240.0,
+      "entropy": 0.07375335693359375,
+      "epoch": 0.9854576561163387,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03276951611042023,
+      "kl": 0.0067653656005859375,
+      "learning_rate": 4.528077426915412e-08,
+      "loss": -0.0088,
+      "num_tokens": 44481928.0,
+      "reward": 0.616766095161438,
+      "reward_std": 0.06268823891878128,
+      "rewards/cross_solution_unittest/mean": 0.6167660355567932,
+      "rewards/cross_solution_unittest/std": 0.09636512398719788,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 108
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -5.0,
+      "completions/max_length": 859.0,
+      "completions/max_terminated_length": 859.0,
+      "completions/mean_length": 480.13543701171875,
+      "completions/mean_terminated_length": 480.13543701171875,
+      "completions/min_length": 205.0,
+      "completions/min_terminated_length": 205.0,
+      "entropy": 0.08184814453125,
+      "epoch": 0.9945822640433419,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.03919151425361633,
+      "kl": 0.007261753082275391,
+      "learning_rate": 2.013323528115674e-08,
+      "loss": 0.0192,
+      "num_tokens": 44857944.0,
+      "reward": 0.6032705903053284,
+      "reward_std": 0.08984307944774628,
+      "rewards/cross_solution_unittest/mean": 0.6032705307006836,
+      "rewards/cross_solution_unittest/std": 0.14218038320541382,
+      "rewards/format_reward/mean": 0.0,
+      "rewards/format_reward/std": 0.0,
+      "step": 109
+    },
+    {
+      "epoch": 0.9945822640433419,
+      "step": 109,
       "total_flos": 0.0,
-      "train_loss": 0.0014987390906618977,
-      "train_runtime": 3075.3471,
-      "train_samples_per_second": 0.428,
-      "train_steps_per_second": 0.009
+      "train_loss": -0.00015112613365182294,
+      "train_runtime": 42785.1182,
+      "train_samples_per_second": 0.031,
+      "train_steps_per_second": 0.003
     }
   ],
   "logging_steps": 1,
-  "max_steps": 28,
-  "num_input_tokens_seen": 5576354,
+  "max_steps": 110,
+  "num_input_tokens_seen": 44857944,
   "num_train_epochs": 1,
-  "save_steps": 50,
+  "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
         "should_epoch_stop": false,
         "should_evaluate": false,
         "should_log": false,
-        "should_save": false,
+        "should_save": true,
         "should_training_stop": false
       },
       "attributes": {}